def run(self):
     "executes sort, accumulate, sort"
     try:
         openlog(self.name, LOG_NDELAY | LOG_CONS | LOG_PID,
                 LOG_LOCAL0)
         q = self._queue
         _sync_pending = Mutex(q._sync_pending_path)
         _sync_pending.acquire()
         pQ = FIFO(q._pQ_path, q._cache_size)
         assert len(pQ) == 0
         pQ_data = os.path.join(q._pQ_sync, "data")
         dumpQ_data = os.path.join(q._dumpQ_sync, "data")
         in_files = [
             os.path.join(pQ_data, cache_file)
             for cache_file in os.listdir(pQ_data)
         ]
         in_files += [
             os.path.join(dumpQ_data, cache_file)
             for cache_file in os.listdir(dumpQ_data)
         ]
         start = time()
         # fast version of chained generators
         map(
             pQ.put,
             q.sort(
                 q.accumulate(
                     q.mergefiles(in_files, (q._unique_key, ))),
                 (q._priority_key, )))
         # slow version of this generator chain:
         #merged_lines = []
         #for x in q.mergefiles(in_files, q._unique_key):
         #    syslog("out of mergefiles: %s" % repr(x))
         #    merged_lines.append(x)
         #accumulated_lines = []
         #for x in q.accumulate(merged_lines):
         #    syslog("out of accumulator: %s" % repr(x))
         #    accumulated_lines.append(x)
         #for x in q.sort(accumulated_lines, q._priority_key):
         #    syslog("out of sort: %s" % repr(x))
         #    pQ.put(x)
         end = time()
         pQ.close()
         syslog(LOG_INFO, "merge took %.1f seconds" % (end - start))
         shutil.rmtree(q._pQ_sync)
         shutil.rmtree(q._dumpQ_sync)
         _sync_pending.release()
     except Exception, exc:
         map(lambda line: syslog(LOG_NOTICE, line),
             traceback.format_exc(exc).splitlines())
class BatchPriorityQueue(MergingRecordFactory):
    """
    A subclass of RecordFactory that maintains two on-disk FIFOs of
    records:

      * _pQ is ordered by records' priority_field lowest-first.
        get(max_priority) retrieves next record from this queue if its
        priority is lower than max_priority.

      * _dumpQ is periodically sorted by unique_field, merged by the
        accumulate function, and sorted again by priority_field in
        order to populate the priority queue.  put() adds records to
        the dump queue.  Calling get() also puts a copy of the record
        into the dump queue, so during the merge, the accumulate
        function will see the previous record and any new records.
        For this reason, records put back into the queue after
        processing a record gotten from the queue should present *only
        changes* relative to the gotten record, so the accumulate
        function can simply add them.

    """
    def __init__(self,
                 record_class,
                 template,
                 data_path,
                 unique_key,
                 priority_key,
                 defaults={},
                 delimiter="|",
                 cache_size=2**16):
        """
        See RecordFactory for record_class, 'template', 'defaults',
        and 'delimiter'.

        'data_path' and 'cache_size' are for _pQ and _dumpQ

        unique_key and priority_key are integer indexes into 'fields'
        indicating which to attributes of records to use as unique
        keys and priorities.
        """
        RecordFactory.__init__(self, record_class, template, defaults,
                               delimiter)
        self._unique_key = unique_key
        self._priority_key = priority_key
        self._cache_size = cache_size
        self._pQ_path = os.path.join(data_path, "pQ")
        self._pQ_sync = os.path.join(data_path, "pQ_sync")
        self._pQ = None
        self._dumpQ_path = os.path.join(data_path, "dumpQ")
        self._dumpQ_sync = os.path.join(data_path, "dumpQ_sync")
        self._dumpQ = FIFO(self._dumpQ_path, self._cache_size)
        # the next deserialized value to return via get
        self._next = None
        self._sync_pending_path = os.path.join(data_path, "sync_pending")
        self._sync_pending = Mutex(self._sync_pending_path)
        self._lock = Mutex(os.path.join(data_path, "lock_file"))

    def close(self):
        """
        Acquires lock, then raises self.Syncing if a sync is in
        progress, otherwise closes internal FIFOs.
        """
        self._lock.acquire()
        try:
            if not self._sync_pending.available():
                raise self.Syncing
            if self._pQ:
                self._pQ.close()
            self._dumpQ.close()
        finally:
            self._lock.release()

    class NotYet(Exception):
        "next record excluded by max_priority"
        pass

    class Blocked(Exception):
        "another process has the mutex"
        pass

    class Syncing(Exception):
        "sync in progress"
        pass

    class ReadyToSync(Exception):
        "pQ empty but records exist in dumpQ"
        pass

    def get(self, max_priority=None, block=True):
        """
        If block=False and cannot acquire lock, raises self.Blocked.

        If a sync is in progress, raises self.Syncing.

        If both _pQ and _dumpQ are empty, then raise Queue.Empty.  

        If empty _pQ but not empty _dumpQ, raise self.ReadyToSync.

        If next item in _pQ has a priority less than max_priority,
        then pops it from queue and returns record.
        """
        acquired = self._lock.acquire(block)
        if not acquired:
            raise self.Blocked
        try:
            if not self._sync_pending.available():
                raise self.Syncing
            if self._pQ is None:
                self._pQ = FIFO(self._pQ_path, self._cache_size)
            if self._next is None:
                # instantiate next record without removing from pQ, raises
                # Queue.Empty when no lines in FIFO
                try:
                    line = self._pQ.next()
                    self._next = self.loads(line)
                except Queue.Empty:
                    if len(self._dumpQ) == 0:
                        raise Queue.Empty
                    else:
                        raise self.ReadyToSync
            if max_priority is None or \
                    self._next[self._priority_key] < max_priority:
                # Remove this line from _pQ and put into _dumpQ. There
                # should be no risk of this raising Queue.Empty.
                self._dumpQ.put(self._pQ.get())
                ret_next = self._next
                self._next = None
                # This is only place that get() returns:
                return ret_next
            elif max_priority is not None:
                raise self.NotYet
            else:
                raise Exception("Should never get here.")
        finally:
            self._lock.release()

    def put(self, record=None, values=None, attrs=None, block=True):
        """
        If record=None, then values or attrs is passed to
        self.create() to obtain a record.

        record is put into _dumpQ.

        If block=False and cannot acquire lock, raises self.Blocked.
        """
        acquired = self._lock.acquire(block)
        if not acquired:
            raise self.Blocked
        if record is None:
            if values is not None:
                record = self.create(*values)
            elif attrs is not None:
                record = self.create(**attrs)
            else:
                raise Exception("put without record, values, or attrs")
        self._dumpQ.put(self.dumps(record))
        self._lock.release()

    def sync(self, block=True):
        """
        Removes all records from _dumpQ and _pQ and performs sort on
        unique_key, accumulate, sort on priority_key before putting
        all records into _pQ.

        If block=False and cannot acquire lock, raises self.Blocked.
        """
        acquired = self._lock.acquire(block)
        if not acquired:
            raise self.Blocked
        try:
            acquired = self._sync_pending.acquire(block=False)
            if not acquired:
                raise self.Syncing
            # move queues to the side
            if self._pQ:
                self._pQ.close()
            self._dumpQ.close()
            os.rename(self._pQ_path, self._pQ_sync)
            os.rename(self._dumpQ_path, self._dumpQ_sync)
            # set pQ to None while syncing, and reopen dumpQ
            self._pQ = None
            self._dumpQ = FIFO(self._dumpQ_path, self._cache_size)
            # Release sync lock momentarily, so merger can acquire it.
            # Get is blocked by _lock, so it won't get confused.
            self._sync_pending.release()
            # launch a child to sort, accumulate, sort
            merger = self.start_merger()
            # loop until merger acquires _sync_pending
            while merger.is_alive() and self._sync_pending.available():
                sleep(0.1)
            # now get back to normal operation
            return merger
        finally:
            self._lock.release()

    def start_merger(self):
        """
        defines, instantiates, and starts a multiprocessing.Process
        for sorting, accumulating, and sorting _dumpQ into new _pQ
        """
        class Merger(multiprocessing.Process):
            "manages the sort, accumulate, sort"
            name = "SortAccumulateSort"
            _queue = self

            def run(self):
                "executes sort, accumulate, sort"
                try:
                    openlog(self.name, LOG_NDELAY | LOG_CONS | LOG_PID,
                            LOG_LOCAL0)
                    q = self._queue
                    _sync_pending = Mutex(q._sync_pending_path)
                    _sync_pending.acquire()
                    pQ = FIFO(q._pQ_path, q._cache_size)
                    assert len(pQ) == 0
                    pQ_data = os.path.join(q._pQ_sync, "data")
                    dumpQ_data = os.path.join(q._dumpQ_sync, "data")
                    in_files = [
                        os.path.join(pQ_data, cache_file)
                        for cache_file in os.listdir(pQ_data)
                    ]
                    in_files += [
                        os.path.join(dumpQ_data, cache_file)
                        for cache_file in os.listdir(dumpQ_data)
                    ]
                    start = time()
                    # fast version of chained generators
                    map(
                        pQ.put,
                        q.sort(
                            q.accumulate(
                                q.mergefiles(in_files, (q._unique_key, ))),
                            (q._priority_key, )))
                    # slow version of this generator chain:
                    #merged_lines = []
                    #for x in q.mergefiles(in_files, q._unique_key):
                    #    syslog("out of mergefiles: %s" % repr(x))
                    #    merged_lines.append(x)
                    #accumulated_lines = []
                    #for x in q.accumulate(merged_lines):
                    #    syslog("out of accumulator: %s" % repr(x))
                    #    accumulated_lines.append(x)
                    #for x in q.sort(accumulated_lines, q._priority_key):
                    #    syslog("out of sort: %s" % repr(x))
                    #    pQ.put(x)
                    end = time()
                    pQ.close()
                    syslog(LOG_INFO, "merge took %.1f seconds" % (end - start))
                    shutil.rmtree(q._pQ_sync)
                    shutil.rmtree(q._dumpQ_sync)
                    _sync_pending.release()
                except Exception, exc:
                    map(lambda line: syslog(LOG_NOTICE, line),
                        traceback.format_exc(exc).splitlines())

        merger = Merger()
        merger.start()
        return merger