예제 #1
0
파일: polyxnjoin.py 프로젝트: Lars-H/slurp
    def probe_tuples_from_source(self, tuple_list, right, ldf_server,
                                 tuple_rjttable):

        probe_ts = time()
        if len(tuple_list) > 0:
            instances = []
            for rtuple in tuple_list:
                instance = {}
                for v in self.vars:
                    instance.update({v: rtuple.data[v]})
                instances.append(instance)

            # Contact the sources.
            qright = Queue()
            right.execute(self.vars, instances, qright, ldf_server=ldf_server)

            # Get the tuples from right_plan queue.
            tuple2 = qright.get(True)
            self.sources = tuple2.sources

            # Empty result set.
            if (tuple2 == "EOF") or (tuple2.data == "EOF"):

                # For all tested tuples add the tail to the records
                for tested_tuple in tuple_list:
                    resource = ''
                    for var in self.vars:
                        resource = resource + str(tested_tuple.data[var])
                    record = Record(tuple2, probe_ts, time(), float("inf"))
                    tail = RJTTail(record, float("inf"))
                    tuple_rjttable[resource] = tail

            # Non-empty result set.
            while (tuple2 != "EOF") and (tuple2.data != "EOF"):

                rtuple_added = False
                for rtuple in tuple_list:

                    if not compatible_solutions(rtuple.data, tuple2.data):
                        continue

                    #print "Got result", rtuple, tuple2, compatible_solutions(rtuple.data, tuple2.data)
                    # Create solution mapping.
                    data = {}
                    data.update(tuple2.data)
                    data.update(rtuple.data)
                    # Update ready and done vectors of solution mapping.
                    ready = tuple2.ready | rtuple.ready
                    done = tuple2.done | rtuple.done | pow(2, self.id_operator)
                    sources = list(set(tuple2.sources) | set(rtuple.sources))

                    # Create tuple.
                    res = Tuple(data, ready, done, sources, self.id_operator)

                    # Introduce the results of contacting the sources in the corresponding table.
                    record = Record(tuple2, probe_ts, time(), float("inf"))
                    resource = ''
                    for var in self.vars:
                        resource = resource + str(rtuple.data[var])

                    # Send tuple to eddy operators.
                    self.to_queue(res, ldf_server)

                    if resource in tuple_rjttable.keys() and not rtuple_added:
                        tuple_rjttable.get(resource).updateRecords(record)
                        tuple_rjttable.get(resource).setRJTProbeTS(probe_ts)
                    else:
                        tail = RJTTail(record, float("inf"))
                        tuple_rjttable[resource] = tail
                        rtuple_added = True

                # Get next solution.
                tuple2 = qright.get(True)

            r_source_id = self.right.source_id
            self.requests[r_source_id] += tuple2.requests.get(r_source_id, 0)
            # Close queue for this sources.
            qright.close()
예제 #2
0
파일: polyfjoin.py 프로젝트: Lars-H/crop
    def probe_tuple(self, tuple_list, right, ldf_server, tuple_rjttable):

        probe_ts = time()
        if len(tuple_list) > 0:
            instances = []
            for rtuple in tuple_list:
                if rtuple.data != "EOF":
                    instance = {}
                    for v in self.vars:
                        instance.update({v: rtuple.data[v]})
                    instances.append(instance)

            if len(instances) > 0:

                # Contact the sources.
                qright = Queue()
                right.execute(self.vars,
                              instances,
                              qright,
                              ldf_server=ldf_server)

                # Get the tuples from right_plan queue.
                tuple2 = qright.get(True)
                self.sources = tuple2.sources

                # Empty result set.
                if (tuple2 == "EOF") or (tuple2.data == "EOF"):

                    # For all tested tuples add the tail to the records
                    for tested_tuple in tuple_list:
                        resource = ''
                        for var in self.vars:
                            resource = resource + str(tested_tuple.data[var])
                        record = Record(tuple2, probe_ts, time(), float("inf"))
                        tail = RJTTail(record, float("inf"))
                        tuple_rjttable[resource] = tail

                # Non-empty result set.
                while (tuple2 != "EOF") and (tuple2.data != "EOF"):

                    rtuple_added = False
                    for rtuple in tuple_list:

                        if not compatible_solutions(rtuple.data, tuple2.data):
                            continue

                        # Create solution mapping.
                        data = {}
                        data.update(tuple2.data)
                        data.update(rtuple.data)
                        # Update ready and done vectors of solution mapping.
                        ready = tuple2.ready | rtuple.ready
                        done = tuple2.done | rtuple.done | pow(
                            2, self.id_operator)
                        sources = list(
                            set(tuple2.sources) | set(rtuple.sources))

                        # Create tuple.
                        res = Tuple(data, ready, done, sources,
                                    self.id_operator)

                        # Introduce the results of contacting the sources in the corresponding table.
                        record = Record(tuple2, probe_ts, time(), float("inf"))
                        resource = ''
                        for var in self.vars:
                            resource = resource + str(rtuple.data[var])

                        # Send tuple to eddy operators.
                        # Send it, if it has not been produced before
                        # TODO: Is this always correct?
                        # What if there are several identical mappings for the same variable from the left_plan side (We
                        # would need to keep track of the triple producing the tuple or remove it from the table)
                        if not res in self.produced_tuples_list:
                            self.to_queue(res)
                            #self.qresults[self.eddy].put(res)

                        if resource in tuple_rjttable.keys(
                        ) and not rtuple_added:
                            tuple_rjttable.get(resource).updateRecords(record)
                            tuple_rjttable.get(resource).setRJTProbeTS(
                                probe_ts)
                        else:
                            tail = RJTTail(record, float("inf"))
                            tuple_rjttable[resource] = tail
                            rtuple_added = True

                    # Get next solution.
                    tuple2 = qright.get(True)

                r_source_id = self.right_operator.source_id
                self.requests[r_source_id] += tuple2.requests.get(
                    r_source_id, 0)
                qright.close()