Exemplo n.º 1
0
def runrealtime():
    try:
        client = Stomp(CONFIG)
        client.connect()
        client.subscribe(QUEUE, {StompSpec.ACK_HEADER: StompSpec.ACK_AUTO})
        log = Logger()
        while True:
            try:
                frame = client.receiveFrame()
                body = json.loads(frame.body.decode())
                plateNumber = body['plateNumber']
                # log.info('Got %s' % json.dumps(body,ensure_ascii=False))
                log.info('Got %s' % plateNumber)
                licenseType = body.get('licenseType', "02")
                log.info('Got licenseType %s' % licenseType)
                log.info("start ..............")
                cjbx_start(plateNumber, licenseType)
                log.info("end ..............")
            except Exception, e:
                log.error(e)
                log.error(frame.body)
                log.error(traceback.format_exc())

    except Exception as e:
        log.error(e)
        log.error(traceback.format_exc())
Exemplo n.º 2
0
def runrealtime():
    try:
        client = Stomp(CONFIG)
        client.connect()
        # client.subscribe(QUEUE, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
        client.subscribe(QUEUE, {StompSpec.ACK_HEADER: StompSpec.ACK_AUTO})
        while True:
            try:
                frame = client.receiveFrame()
                # body=json.dumps(frame.body)
                body = json.loads(frame.body.decode())[0]
                insuranceType = body.get("insuranceType", [])[0]
                body['insuranceType'] = insuranceType
                companyId = body.get('companyId', [])
                #保险公司选择
                if len(companyId) == 0 or "2" in companyId:
                    client.send(body=json.dumps(body,
                                                ensure_ascii=False).encode(),
                                destination=EPICC)
                log.info('Got %s' % json.dumps(body, ensure_ascii=False))
                # client.ack(frame)
                body['client'] = client
                getSource(body)
            except Exception as e:
                log.error(e)
                log.error(frame.body)
                log.error(traceback.format_exc())

    except Exception as e:
        log.error(e)
        log.error(traceback.format_exc())
Exemplo n.º 3
0
def runrealtime():
    try:
        client = Stomp(CONFIG)
        client.connect()
        #client.subscribe(QUEUE, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
        client.subscribe(BATCH_EPICC_QUEUE, {StompSpec.ACK_HEADER: StompSpec.ACK_AUTO})
        while True:
            try:
                frame = client.receiveFrame()
                body=json.loads(frame.body)
                log.info('Got %s' % json.dumps(body,ensure_ascii=False))
                epicc_batch(body,client)
            except Exception,e:
                log.error(e)
                log.error(frame.body)
                log.error(traceback.format_exc())

    except Exception as e:
            log.error(e)
            log.error(traceback.format_exc())
Exemplo n.º 4
0
def runrealtime():
    try:
        client = Stomp(CONFIG)
        client.connect()
        # client.subscribe(QUEUE, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL})
        client.subscribe(BATCH_PROCESS_QUEUE,
                         {StompSpec.ACK_HEADER: StompSpec.ACK_AUTO})
        while True:
            try:
                frame = client.receiveFrame()
                body = json.loads(frame.body.decode())
                client.send(body=json.dumps(body, ensure_ascii=False).encode(),
                            destination=se.EPICC_BATCH_REPAIR_QUEUE)
                batch_repair_thread(body, client)
            except Exception, e:
                log.error(e)
                log.error(frame.body)
                log.error(traceback.format_exc())

    except Exception as e:
        log.error(e)
        log.error(traceback.format_exc())
Exemplo n.º 5
0
class Command(BaseCommand):
    """Service that listens for Fedora STOMP updates and processes those objects
       against the configured site indexes."""
    to_index = {}
    # delay before indexing
    # - since an object may be updated by several API calls in sequence, delay a bit
    #   so that hopefully we only index it once for a single group of updates
    # TODO: make delay configurable
    index_delay = 4  # time delay after the *first* modification message before indexing should be done
    index_delta = timedelta(seconds=index_delay)
    index_max_tries = 3  # number of times to try indexing an item, if a recoverable error happens

    stomp_server = None
    stomp_port = None
    listener = None

    # connection error handling/retry settings
    retry_reconnect_wait = 5
    # if we lose the connection to fedora, how long do we wait between attempts to reconnect?
    max_reconnect_retries = 5
    # how many times do we try to reconnect to fedora before we give up?

    idle_reconnect = None

    option_list = BaseCommand.option_list + (
        make_option('--max-reconnect-retries', type='int', dest='max_reconnect_retries',
                    default=max_reconnect_retries,
                    help='How many times to try reconnecting if the connection ' +
                    	 'is lost (default: %default; -1 for no maximum)'),
        make_option('--retry-reconnect-wait', type='int', dest='retry_reconnect_wait',
                    default=retry_reconnect_wait,
                    help='How many seconds to wait between reconnect attempts if the ' +
                    	 'connection is lost (default: %default)'),
        make_option('--index-max-tries', type='int', dest='index_max_tries',
                    default=index_max_tries,
                    help='Number of times to attempt indexing an item when a potentially ' +
                    	 'recoverable error is encountered'),
        make_option('--idle-reconnect', type='int', dest='idle_reconnect',
                    help='Reconnect when there has been no activity for the specified ' +
                         'number of minutes')

    )

    # flag will be set to True when a SIGINT has been received
    interrupted = False

    # class variables defined in setup
    indexes = []
    last_activity = None
    verbosity = None
    repo = None

    def init_listener(self):
        if self.stomp_server is None or self.stomp_port is None:
            self.stomp_server = settings.INDEXER_STOMP_SERVER
            self.stomp_port = int(settings.INDEXER_STOMP_PORT)

        self.listener = Stomp(StompConfig('tcp://%s:%s' % (self.stomp_server, self.stomp_port)))
        self.listener.connect()
        logger.info('Connected to message queue on %s:%i',
                    self.stomp_server, self.stomp_port)
        self.listener.subscribe(settings.INDEXER_STOMP_CHANNEL,
                                {'ack': 'client'})  #  can we use auto-ack ?
        self.last_activity = datetime.now()

    def init_indexes(self):
        # initialize all indexes configured in django settings
        self.indexes, init_errors = init_configured_indexes()
        if init_errors:
            msg = 'Error loading index configuration for the following sites:\n'
            for site, err in init_errors.iteritems():
                msg += '\t%s:\t%s\n' % (site, err)
                self.stdout.write(msg + '\n')

        if self.verbosity > self.v_normal:
            self.stdout.write('Indexing the following sites:\n')
            for site, index in self.indexes.iteritems():
                self.stdout.write('\t%s\n%s\n' % (site, index.config_summary()))

    # verbosity option set by django BaseCommand
    v_normal = 1	    # 1 = normal, 0 = minimal, 2 = all

    def handle(self, verbosity=v_normal, retry_reconnect_wait=None,
               max_reconnect_retries=None,  index_max_tries=None,
               idle_reconnect=None, *args, **options):
        # bind a handler for interrupt signal
        signal.signal(signal.SIGINT, self.interrupt_handler)
        signal.signal(signal.SIGHUP, self.hangup_handler)

        # verbosity should be set by django BaseCommand standard options
        self.verbosity = int(verbosity)

        # override retry/wait default settings if specified
        if retry_reconnect_wait:
            self.retry_reconnect_wait = retry_reconnect_wait
        if max_reconnect_retries:
            self.max_reconnect_retries = max_reconnect_retries
        if index_max_tries:
            self.index_max_tries = index_max_tries

        if idle_reconnect is not None:
            self.idle_reconnect = timedelta(minutes=idle_reconnect)

        # check for required settings

        self.repo = Repository()
        try:
            self.init_listener()
        except StompConnectTimeout:
            # if we can't connect on start-up, bail out
            raise CommandError('Error connecting to %s:%s' % (self.stomp_server, self.stomp_port) +
                               '\nCheck that Fedora is running and that messaging is enabled ' +
                               'and configured correctly')

        # load site index configurations
        self.init_indexes()

        while (True):

            # check time since last activity if idle reconnect is configured
            if self.idle_reconnect and \
                   datetime.now() - self.last_activity >= self.idle_reconnect:
                logger.info('Time since last activity has exceeded idle reconnect time [%s]',
                            self.idle_reconnect)
                # disconnect and reconnect the stomp listener
                self.listener.disconnect()
                self.reconnect_listener()


            # if we've received an interrupt, don't check for new messages
            if self.interrupted:
                # if we're interrupted but still have items queued for update,
                # sleep instead of listening to the queue
                if self.to_index:
                    sleep(self.index_delay)

            # no interrupt - normal behavior
            # check if there is a new message, but timeout after 3 seconds so we can process
            # any recently updated objects
            else:
                # check if there is a new message, but timeout after 3 seconds so we can process
                # any recently updated objects
                try:
                    data_available = self.listener.canRead(timeout=self.index_delay)

                except StompConnectionError as err:
                    # probably indicates fedora has gone down
                    logger.error('Received Stomp Connection error "%s"',  err)
                    self.reconnect_listener()
                    data_available = False

                except Exception as err:
                    # signals like SIGINT/SIGHUP get propagated to the socket
                    if self.interrupted:
                        pass
                    else:
                        logger.error('Error during Stomp listen: %s', err)
                    data_available = False

                # When Fedora is shut down, canRead returns True but we
                # get an exception on the receiveFrame call - catch that
                # error and try to reconnect
                try:

                    # if there is a new message, process it
                    if data_available:
                        frame = self.listener.receiveFrame()
                        self.last_activity = datetime.now()
                        # NOTE: could make use of message body instead of/in addition to headers
                        # (includes datastream id for modify datastream API calls)
                        pid = frame.headers['pid']
                        method = frame.headers['methodName']
                        logger.debug('Received message: %s %s', method, pid)
                        self.listener.ack(frame)

                        self.process_message(pid, method)

                except Exception as e:
                    # this most likely indicates that Fedora is no longer available;
                    # periodically attempt to reconnect (within some limits)
                    if isinstance(e, StompFrameError):
                        logger.error('Received Stomp frame error "%s"',  e)
                    else:
                        # in some cases, getting a generic Exception "Connection Closed"
                        # when Fedora shuts down
                        logger.error('Error listening to Stomp: %s', e)

                    # wait and try to re-establish the listener
                    # - will either return on success or raise a CommandError if
                    # it can't connect within the specified time/number of retries
                    self.reconnect_listener()

            #Process the index queue for any items that need it.
            self.process_queue()

            # if we've received an interrupt and there is nothing queued to index,
            # quit
            if self.interrupted and not self.to_index:
                return

    def reconnect_listener(self):
        '''Attempt to reconnect the listener, e.g. if Fedora is
        shutdown.  Waits the configured time between attemps to
        reconnect; will try to reconnect a configured number of times
        before giving up.'''

        # wait the configured time and try to re-establish the listener
        retry_count = 1
        while(retry_count <= self.max_reconnect_retries or self.max_reconnect_retries == -1):
            sleep(self.retry_reconnect_wait)
            try:
                self.listener = None
                self.init_listener()
                # if listener init succeeded, return for normal processing
                logger.info('Reconnect attempt %d succeeded', retry_count)
                return

            # if fedora is still not available, attempting to
            # listen will generate a socket error
            except StompError:
                # could be StompConnectTimeout, StompConnectionError, etc.

                try_detail = ''
                if self.max_reconnect_retries != -1:
                    try_detail = 'of %d ' % self.max_reconnect_retries
                logger.error('Reconnect attempt %d %sfailed; waiting %ds before trying again',
                             retry_count, try_detail, self.retry_reconnect_wait)
                retry_count += 1

        # if we reached the max retry without connecting, bail out
        # TODO: better error reporting - should this send an admin email?
        raise CommandError('Failed to reconnect to message queue after %d retries',
                           (retry_count - 1))

    def process_message(self, pid, method):
        # process an update message from fedora

        # when an object is purged from fedora, remove it from the index
        if method == 'purgeObject':
            # since we don't know which index (if any) this object was indexed in,
            # delete it from all configured indexes
            for site, index in self.indexes.iteritems():
                try:
                    index.delete_item(pid)
                except Exception as e:
                    logging.error("Failed to purge %s (%s): %s",
                                  pid, site, e)

                    # Add a prefix to the detail error message if we
                    # can identify what type of error this is.
                    detail_type = ''
                    if isinstance(e, SolrError):
                        detail_type = 'Solr Error: '
                    action_str = 'Purge: '
                    msg = '%s%s%s' % (detail_type, action_str, e)
                    err = IndexError(object_id=pid, site=site, detail=msg)
                    err.save()
            logger.info('Deleting %s from all configured Solr indexes', pid)
            # commit?

        # ingest, modify object or modify datastream
        else:
            # if the object isn't already in the queue to be indexed, check if it should be
            if pid not in self.to_index:
                # get content models from resource index
                obj_cmodels = list(self.repo.risearch.get_objects('info:fedora/%s' % pid,
                                                                  modelns.hasModel))
                # may include generic content models, but should not be a problem

                # find which configured site(s) index the item
                for site, index in self.indexes.iteritems():
                    if index.indexes_item(obj_cmodels):
                        if pid not in self.to_index:
                            # first site found - create a queue item and add to the list
                            self.to_index[pid] = QueueItem(site)
                        else:
                            # subsequent site - add the site to the existing queue item
                            self.to_index[pid].add_site(site)

    def process_queue(self):
        '''Loop through items that have been queued for indexing; if
        the configured delay time has passed, then attempt to index
        them, and log any indexing errors.'''
        #check if there are any items that should be indexed now
        if self.to_index:
            logger.debug('Objects queued to be indexed: %s',
                         ', '.join(self.to_index.keys()))

            queue_remove = []
            for pid in self.to_index.iterkeys():
                # if we've waited the configured delay time, attempt to index
                if datetime.now() - self.to_index[pid].time >= self.index_delta:
                    sites_to_index = self.to_index[pid].sites_to_index

                    logger.info('Indexing %s in %s',
                        pid, ', '.join(sites_to_index))

                    # a single object could be indexed by multiple sites; index all of them
                    for site in sites_to_index:
                        self.index_item(pid, self.to_index[pid], site)

                    if not self.to_index[pid].sites_to_index:
                        # if all configured sites indexed successfully
                        # or failed and should not be re-indexed,
                        # store pid to be removed from the queue
                        queue_remove.append(pid)


            # clear out any pids that were indexed successfully OR
            # errored from the list of objects still to be indexed
            for pid in queue_remove:
                del self.to_index[pid]

    def index_item(self, pid, queueitem, site):
        '''Index an item in a single configured site index and handle
        any errors, updating the queueitem retry count and marking
        sites as indexed according to success or any errors.

        :param pid: pid for the item to be indexed
        :param queueitem: :class:`QueueItem`
        :param site: name of the site index to use
        '''
        try:
            # tell the site index to index the item - returns True on success
            if self.indexes[site].index_item(pid):
                # mark the site index as complete on the queued item
                self.to_index[pid].site_complete(site)

        except RecoverableIndexError as rie:
            # If the index attempt resulted in error that we
            # can potentially recover from, keep the item in
            # the queue and attempt to index it again.

            # Increase the count of index attempts, so we know when to stop.
            self.to_index[pid].tries += 1

            # quit when we reached the configured number of index attempts
            if self.to_index[pid].tries >= self.index_max_tries:
                logger.error("Failed to index %s (%s) after %d tries: %s",
                              pid, site, self.to_index[pid].tries, rie)

                err = IndexError(object_id=pid, site=site,
                                 detail='Failed to index after %d attempts: %s' % \
                                 (self.to_index[pid].tries, rie))
                err.save()
                # we've hit the index retry limit, so set site as complete on the queue item
                self.to_index[pid].site_complete(site)

            else:
                logging.warn("Recoverable error attempting to index %s (%s), %d tries: %s",
                             pid, site, self.to_index[pid].tries, rie)

                # update the index time - wait the configured index delay before
                # attempting to reindex again
                self.to_index[pid].time = datetime.now()

        except Exception as e:
            logging.error("Failed to index %s (%s): %s",
                          pid, site, e)

            # Add a prefix to the detail error message if we
            # can identify what type of error this is.
            detail_type = ''
            if isinstance(e, SolrError):
                detail_type = 'Solr Error: '
            msg = '%s%s' % (detail_type, e)
            err = IndexError(object_id=pid, site=site,
                             detail=msg)
            err.save()

            # any exception not caught in the recoverable error block
            # should not be attempted again - set site as complete on queue item
            self.to_index[pid].site_complete(site)


    def interrupt_handler(self, signum, frame):
        '''Gracefully handle a SIGINT, if possible.  Reports status if
        main loop is currently part-way through pages for a volume,
        sets a flag so main script loop can exit cleanly, and restores
        the default SIGINT behavior, so that a second interrupt will
        stop the script.
        '''
        if signum == signal.SIGINT:
            # restore default signal handler so a second SIGINT can be used to quit
            signal.signal(signal.SIGINT, signal.SIG_DFL)
            # set interrupt flag so main loop knows to quit as soon as it can
            self.interrupted = True

            if self.verbosity >= self.v_normal:
                # log as well as printing to stdout (for manual run & init.d-type service run)
                msg = 'SIGINT received; stopping'
                self.stdout.write('%s\n' % msg)
                logger.info(msg)

            # report if indexer currently has items queued for indexing
            if self.to_index:
                msg = '\n%d item(s) currently queued for indexing.\n' % len(self.to_index.keys())
                msg += 'Indexer will stop listening for updates and exit after currently queued items have been indexed.\n'
                # log summary information
                logger.info(msg)
                msg += '(Ctrl-C / Interrupt again to quit immediately)\n'
                self.stdout.write(msg)

            self.stdout.flush()

    def hangup_handler(self, signum, frame):
        '''On SIGHUP, reload site index configurations and
        reinitialize connections to Solr.
        '''
        if signum == signal.SIGHUP:
            # it would be even better if we could reload django settings here,
            # but I can't figure out a way to do that...

            if self.verbosity >= self.v_normal:
                # log as well as printing to stdout
                msg = 'SIGHUP received; reloading site index configurations.'
                self.stdout.write(msg)
                logger.info(msg)
            # reload site indexes
            self.init_indexes()