Beispiel #1
0
 def __init__(self, theory, region_size, region_queue_size, **options):
     self.options = options
     self.log_file = options['log_file']
     self.world = DB('world')
     self.normal_world = DB('world.normal')
     self.normal_region = DB('region.normal.{:d}')
     self.min_size = pomagma.util.MIN_SIZES[theory]
     self.region_size = region_size
     self.region_queue = FileQueue('region.queue')
     self.survey_queue = FileQueue('survey.queue')
     self.region_queue_size = region_queue_size
     self.diverge_conjectures = 'diverge_conjectures.facts'
     self.diverge_theorems = 'diverge_theorems.facts'
     self.equal_conjectures = 'equal_conjectures.facts'
     DEBUG = False
     if DEBUG:
         options = pomagma.util.use_memcheck(
             options,
             'cartographer.memcheck.out')
     self.server = pomagma.cartographer.serve(theory, self.world, **options)
     self.db = self.server.connect()
     self.infer_state = 0
     if os.path.exists(self.normal_world):
         world_digest = pomagma.atlas.get_hash(self.world)
         normal_world_digest = pomagma.atlas.get_hash(self.normal_world)
         if world_digest == normal_world_digest:
             self.infer_state = 2
Beispiel #2
0
class CartographerWorker(object):

    def __init__(self, theory, region_size, region_queue_size, **options):
        self.options = options
        self.log_file = options['log_file']
        self.world = DB('world')
        self.normal_world = DB('world.normal')
        self.normal_region = DB('region.normal.{:d}')
        self.min_size = pomagma.util.MIN_SIZES[theory]
        self.region_size = region_size
        self.region_queue = FileQueue('region.queue')
        self.survey_queue = FileQueue('survey.queue')
        self.region_queue_size = region_queue_size
        self.diverge_conjectures = 'diverge_conjectures.facts'
        self.diverge_theorems = 'diverge_theorems.facts'
        self.equal_conjectures = 'equal_conjectures.facts'
        DEBUG = False
        if DEBUG:
            options = pomagma.util.use_memcheck(
                options,
                'cartographer.memcheck.out')
        self.server = pomagma.cartographer.serve(theory, self.world, **options)
        self.db = self.server.connect()
        self.infer_state = 0
        if os.path.exists(self.normal_world):
            world_digest = pomagma.atlas.get_hash(self.world)
            normal_world_digest = pomagma.atlas.get_hash(self.normal_world)
            if world_digest == normal_world_digest:
                self.infer_state = 2

    def stop(self):
        self.db.stop()
        # self.server.stop()

    def log(self, message):
        rss = pomagma.util.get_rss(self.server.pid)
        message = 'Cartographer {}k {}'.format(rss, message)
        pomagma.util.log_print(message, self.log_file)

    def is_normal(self):
        assert self.infer_state in [0, 1, 2]
        return self.infer_state == 2

    def garbage_collect(self):
        # assume surveyor.dump takes < 1.0 days
        pomagma.atlas.garbage_collect(grace_period_days=1.0)

    def try_work(self):
        return (
            self.try_produce_regions() or
            self.try_normalize() or
            self.try_consume_surveys()
        )

    def try_produce_regions(self):
        queue_size = len(self.region_queue)
        if queue_size >= self.region_queue_size:
            return False
        else:
            self.fill_region_queue(self.region_queue)
            return True

    def try_normalize(self):
        if self.is_normal():
            return False
        else:
            self.log('Inferring {}'.format(['pos', 'neg'][self.infer_state]))
            if self.db.infer(self.infer_state):
                self.db.validate()
                self.db.dump(self.world)
                self.garbage_collect()
                self.replace_region_queue()
            else:
                self.infer_state += 1
                if self.is_normal():
                    self.log('Normalized')
                    self.db.dump(self.normal_world)
                    self.trim_normal_regions()
                    self.garbage_collect()
                    self.theorize()
            return True

    def try_consume_surveys(self):
        surveys = self.survey_queue.get()
        if not surveys:
            return False
        else:
            self.log('Aggregating {} surveys'.format(len(surveys)))
            for survey in surveys:
                self.db.aggregate(survey)
                self.db.validate()
                self.db.dump(self.world)
                self.garbage_collect()
                self.infer_state = 0
                world_size = self.db.info()['item_count']
                self.log('world_size = {}'.format(world_size))
                os.remove(survey)
            self.db.crop()
            self.replace_region_queue()
            return True

    def fill_region_queue(self, queue):
        self.log('Filling region queue')
        if not os.path.exists(queue.path):
            os.makedirs(queue.path)
        queue_size = len(queue)
        trim_count = max(0, self.region_queue_size - queue_size)
        regions_out = []
        for i in itertools.count():
            region_out = os.path.join(queue.path, DB(i))
            if not os.path.exists(region_out):
                regions_out.append(region_out)
                if len(regions_out) == trim_count:
                    break
        # trim in parallel because these are small
        self.db.trim([
            {'size': self.region_size, 'filename': r}
            for r in regions_out
        ])

    def replace_region_queue(self):
        self.log('Replacing region queue')
        with pomagma.util.temp_copy(self.region_queue.path) as temp_path:
            self.fill_region_queue(FileQueue(temp_path))
            self.region_queue.clear()
            self.garbage_collect()

    def trim_normal_regions(self):
        self.log('Trimming normal regions')
        assert self.is_normal()
        max_size = self.db.info()['item_count']
        # trim sequentially because these are large
        for size in suggest_region_sizes(self.min_size, max_size):
            filename = self.normal_region.format(size)
            self.db.trim([{'size': size, 'filename': filename}])

    def theorize(self):
        self.log('Theorizing')
        conjectures = self.diverge_conjectures
        theorems = self.diverge_theorems
        self.db.conjecture(conjectures, self.equal_conjectures)
        with pomagma.util.temp_copy(conjectures) as temp_conjectures:
            with pomagma.util.temp_copy(theorems) as temp_theorems:
                if os.path.exists(theorems):
                    shutil.copyfile(theorems, temp_theorems)
                theorem_count = pomagma.theorist.try_prove_diverge(
                    conjectures,
                    temp_conjectures,
                    temp_theorems,
                    **self.options)
        if theorem_count > 0:
            self.log('Proved {} theorems'.format(theorem_count))
            counts = self.db.assume(theorems)
            if counts['pos'] + counts['neg']:
                self.log('Assumed {} pos + {} neg facts'.format(
                    counts['pos'],
                    counts['neg']))
                self.db.validate()
                self.db.dump(self.world)
                self.garbage_collect()
                self.infer_state = 0 if counts['pos'] else 1
                self.replace_region_queue()