Exemple #1
0
    def stage1(self, tuple, tuple_rjttable, other_rjttable, vars):
        # Stage 1: While one of the sources is sending data.

        # Get the value(s) of the operator variable(s) in the tuple.
        resource = ''
        if tuple.data != "EOF":
            for var in self.vars:
                try:
                    resource = resource + str(tuple.data[var])
                    # print("{}: {}".format(self.id_operator, str(tuple1.data)))
                except Exception as e:
                    raise e
        else:
            resource = "EOF"

        # Probe the tuple against its RJT table.
        probe_ts = self.probe(tuple, resource, tuple_rjttable)

        # Create the records.
        record = Record(tuple, probe_ts, time(), float("inf"))

        # Insert the record in the corresponding RJT table.
        if resource in other_rjttable:
            other_rjttable.get(resource).updateRecords(record)
            other_rjttable.get(resource).setRJTProbeTS(probe_ts)
        else:
            tail = RJTTail(record, probe_ts)
            other_rjttable[resource] = tail
Exemple #2
0
    def stage1_hash_join(self, tuple1, tuple_rjttable, other_rjttable):

        # Get the value(s) of the operator variable(s) in the tuple.
        resource = ''
        if tuple1.data != "EOF":
            for var in self.vars:
                try:
                    resource = resource + str(tuple1.data[var])
                except Exception as e:
                    raise e
        else:
            resource = "EOF"

        # Probe the tuple against its RJT table.
        probe_ts = self.probe_hash_table(tuple1, resource, tuple_rjttable)

        # Create the records.
        record = Record(tuple1, probe_ts, time(), float("inf"))

        # Insert the record in the corresponding RJT table.
        if resource in other_rjttable:
            other_rjttable.get(resource).updateRecords(record)
            other_rjttable.get(resource).setRJTProbeTS(probe_ts)
        else:
            tail = RJTTail(record, probe_ts)
            other_rjttable[resource] = tail
Exemple #3
0
    def stage1(self, tuple, tuple_rjttable, other_rjttable):
        # Stage 1: While one of the sources is sending data.

        # Get the resource associated to the tuples.
        if tuple.data != "EOF" and tuple != "EOF":
            resource = ''
            for var in self.vars:
                resource = resource + str(tuple.data[var])

            # Probe the tuple against its RJT table.
            probeTS = self.probe(tuple, resource, tuple_rjttable,
                                 other_rjttable)

            # Create the records.
            record = Record(tuple, probeTS, time(), float("inf"))

            # Insert the record in the other RJT table.
            # TODO: use RJTTail. Check ProbeTS
            if resource in other_rjttable:
                other_rjttable.get(resource).updateRecords(record)
                other_rjttable.get(resource).setRJTProbeTS(probeTS)
                #other_rjttable.get(resource).append(record)
            else:
                tail = RJTTail(record, float("inf"))
                other_rjttable[resource] = tail
Exemple #4
0
    def stage1(self, tuple1, tuple_rjttable, other_rjttable):

        if tuple1.data != "EOF" and tuple1 != "EOF":

            # Get the value(s) of the operator variable(s) in the tuple.
            resource = ''
            for var in self.vars:
                resource = resource + str(tuple1.data[var])

            # Probe the tuple against its RJT table.
            probe_ts = self.probe(tuple1, resource, tuple_rjttable,
                                  other_rjttable)

            # Create the record.
            record = Record(tuple1, probe_ts, time(), float("inf"))

            # Insert the record in the other RJT table.
            # TODO: use RJTTail. Check ProbeTS
            if resource in other_rjttable:
                other_rjttable.get(resource).updateRecords(record)
                other_rjttable.get(resource).setRJTProbeTS(probe_ts)
            else:
                tail = RJTTail(record, float("inf"))
                other_rjttable[resource] = tail
Exemple #5
0
    def probe(self, tuple1, resource, rjttable, other_rjttable):

        probe_ts = time()

        # If the resource is in table, produce results.
        if resource in rjttable.keys():

            rjttable.get(resource).setRJTProbeTS(probe_ts)
            list_records = rjttable[resource].records

            # For each match, produce the results (solution mappings).
            for record in list_records:
                res = {}

                if record.tuple.data == "EOF":
                    break

                # Merge solution mappings.
                res.update(record.tuple.data)
                res.update(tuple1.data)

                # Update ready and done vectors.
                ready = record.tuple.ready | tuple1.ready
                done = record.tuple.done | tuple1.done | pow(
                    2, self.id_operator)
                sources = list(set(record.tuple.sources) | set(tuple1.sources))

                # Create solution mapping.
                res = Tuple(res, ready, done, sources, self.id_operator)

                # Send solution mapping to eddy operators.
                self.to_queue(res)
                #self.qresults[self.eddy].put(res)

        # If the resource is not in the table, contact the sources.
        else:

            # Extract domain and range of operator variables from the tuple.
            instances = {}
            for v in self.vars:
                instances.update({v: tuple1.data[v]})

            # Contact the sources.
            qright = Queue()
            self.right.execute(self.vars, instances, qright)

            # Get the tuples from right_plan queue.
            tuple2 = qright.get(True)
            self.sources = tuple2.sources

            # Empty result set.
            if (tuple2 == "EOF") or (tuple2.data == "EOF"):

                record = Record(tuple2, probe_ts, time(), float("inf"))
                tail = RJTTail(record, float("inf"))
                rjttable[resource] = tail

            # Non-empty result set.
            while (tuple2 != "EOF") and (tuple2.data != "EOF"):

                # Create solution mapping.
                data = {}
                data.update(tuple2.data)
                data.update(tuple1.data)

                #print("{}; {}".format(self.id_operator, data))
                # Update ready and done vectors of solution mapping.
                ready = tuple2.ready | tuple1.ready
                done = tuple2.done | tuple1.done | pow(2, self.id_operator)
                sources = list(set(tuple2.sources) | set(tuple1.sources))

                # Create tuple.
                res = Tuple(data, ready, done, sources, self.id_operator)

                # Send tuple to eddy operators.
                self.to_queue(res)
                #self.qresults[self.eddy].put(res)

                # Introduce the results of contacting the sources in the corresponding table.
                record = Record(tuple2, probe_ts, time(), float("inf"))
                if resource in rjttable.keys():
                    rjttable.get(resource).updateRecords(record)
                    rjttable.get(resource).setRJTProbeTS(probe_ts)
                else:
                    tail = RJTTail(record, float("inf"))
                    rjttable[resource] = tail

                # Get next solution.
                tuple2 = qright.get(True)

            # Close queue for this sources.
            qright.close()

        return probe_ts
Exemple #6
0
    def probe_tuples_from_source(self, tuple_list, right, ldf_server,
                                 tuple_rjttable):

        probe_ts = time()
        if len(tuple_list) > 0:
            instances = []
            for rtuple in tuple_list:
                instance = {}
                for v in self.vars:
                    instance.update({v: rtuple.data[v]})
                instances.append(instance)

            # Contact the sources.
            qright = Queue()
            right.execute(self.vars, instances, qright, ldf_server=ldf_server)

            # Get the tuples from right_plan queue.
            tuple2 = qright.get(True)
            self.sources = tuple2.sources

            # Empty result set.
            if (tuple2 == "EOF") or (tuple2.data == "EOF"):

                # For all tested tuples add the tail to the records
                for tested_tuple in tuple_list:
                    resource = ''
                    for var in self.vars:
                        resource = resource + str(tested_tuple.data[var])
                    record = Record(tuple2, probe_ts, time(), float("inf"))
                    tail = RJTTail(record, float("inf"))
                    tuple_rjttable[resource] = tail

            # Non-empty result set.
            while (tuple2 != "EOF") and (tuple2.data != "EOF"):

                rtuple_added = False
                for rtuple in tuple_list:

                    if not compatible_solutions(rtuple.data, tuple2.data):
                        continue

                    #print "Got result", rtuple, tuple2, compatible_solutions(rtuple.data, tuple2.data)
                    # Create solution mapping.
                    data = {}
                    data.update(tuple2.data)
                    data.update(rtuple.data)
                    # Update ready and done vectors of solution mapping.
                    ready = tuple2.ready | rtuple.ready
                    done = tuple2.done | rtuple.done | pow(2, self.id_operator)
                    sources = list(set(tuple2.sources) | set(rtuple.sources))

                    # Create tuple.
                    res = Tuple(data, ready, done, sources, self.id_operator)

                    # Introduce the results of contacting the sources in the corresponding table.
                    record = Record(tuple2, probe_ts, time(), float("inf"))
                    resource = ''
                    for var in self.vars:
                        resource = resource + str(rtuple.data[var])

                    # Send tuple to eddy operators.
                    self.to_queue(res, ldf_server)

                    if resource in tuple_rjttable.keys() and not rtuple_added:
                        tuple_rjttable.get(resource).updateRecords(record)
                        tuple_rjttable.get(resource).setRJTProbeTS(probe_ts)
                    else:
                        tail = RJTTail(record, float("inf"))
                        tuple_rjttable[resource] = tail
                        rtuple_added = True

                # Get next solution.
                tuple2 = qright.get(True)

            r_source_id = self.right.source_id
            self.requests[r_source_id] += tuple2.requests.get(r_source_id, 0)
            # Close queue for this sources.
            qright.close()
Exemple #7
0
    def probe_tuple(self, tuple_list, right, ldf_server, tuple_rjttable):

        probe_ts = time()
        if len(tuple_list) > 0:
            instances = []
            for rtuple in tuple_list:
                if rtuple.data != "EOF":
                    instance = {}
                    for v in self.vars:
                        instance.update({v: rtuple.data[v]})
                    instances.append(instance)

            if len(instances) > 0:

                # Contact the sources.
                qright = Queue()
                right.execute(self.vars,
                              instances,
                              qright,
                              ldf_server=ldf_server)

                # Get the tuples from right_plan queue.
                tuple2 = qright.get(True)
                self.sources = tuple2.sources

                # Empty result set.
                if (tuple2 == "EOF") or (tuple2.data == "EOF"):

                    # For all tested tuples add the tail to the records
                    for tested_tuple in tuple_list:
                        resource = ''
                        for var in self.vars:
                            resource = resource + str(tested_tuple.data[var])
                        record = Record(tuple2, probe_ts, time(), float("inf"))
                        tail = RJTTail(record, float("inf"))
                        tuple_rjttable[resource] = tail

                # Non-empty result set.
                while (tuple2 != "EOF") and (tuple2.data != "EOF"):

                    rtuple_added = False
                    for rtuple in tuple_list:

                        if not compatible_solutions(rtuple.data, tuple2.data):
                            continue

                        # Create solution mapping.
                        data = {}
                        data.update(tuple2.data)
                        data.update(rtuple.data)
                        # Update ready and done vectors of solution mapping.
                        ready = tuple2.ready | rtuple.ready
                        done = tuple2.done | rtuple.done | pow(
                            2, self.id_operator)
                        sources = list(
                            set(tuple2.sources) | set(rtuple.sources))

                        # Create tuple.
                        res = Tuple(data, ready, done, sources,
                                    self.id_operator)

                        # Introduce the results of contacting the sources in the corresponding table.
                        record = Record(tuple2, probe_ts, time(), float("inf"))
                        resource = ''
                        for var in self.vars:
                            resource = resource + str(rtuple.data[var])

                        # Send tuple to eddy operators.
                        # Send it, if it has not been produced before
                        # TODO: Is this always correct?
                        # What if there are several identical mappings for the same variable from the left_plan side (We
                        # would need to keep track of the triple producing the tuple or remove it from the table)
                        if not res in self.produced_tuples_list:
                            self.to_queue(res)
                            #self.qresults[self.eddy].put(res)

                        if resource in tuple_rjttable.keys(
                        ) and not rtuple_added:
                            tuple_rjttable.get(resource).updateRecords(record)
                            tuple_rjttable.get(resource).setRJTProbeTS(
                                probe_ts)
                        else:
                            tail = RJTTail(record, float("inf"))
                            tuple_rjttable[resource] = tail
                            rtuple_added = True

                    # Get next solution.
                    tuple2 = qright.get(True)

                r_source_id = self.right_operator.source_id
                self.requests[r_source_id] += tuple2.requests.get(
                    r_source_id, 0)
                qright.close()
Exemple #8
0
    def probe(self, tuple, resource, rjttable, other_rjttable):
        probeTS = time()

        # If the resource is in table, produce results.
        if resource in rjttable:
            rjttable.get(resource).setRJTProbeTS(probeTS)
            list_records = rjttable[resource].records
            #list_records = rjttable[resource]

            # For each match, produce the results (solution mappings).
            for record in list_records:
                res = {}

                if record.tuple.data == "EOF":
                    break

                # Merge solution mappings.
                res.update(record.tuple.data)
                res.update(tuple.data)

                # Update ready and done vectors.
                ready = record.tuple.ready | tuple.ready
                done = record.tuple.done | tuple.done | pow(
                    2, self.id_operator)
                sources = list(set(record.tuple.sources) | set(tuple.sources))

                # Create solution mapping.
                res = Tuple(res, ready, done, sources, self.id_operator)

                # Send solution mapping to eddy operators.
                self.qresults[self.eddy].put(res)

        # If not, contact the source.
        else:
            instances = {}
            for v in self.vars:
                instances.update({v: tuple.data[v]})

            # Contact the source.
            qright = Queue()
            self.right.execute(self.vars, instances, qright)

            # Get the tuples from right_plan queue.
            rtuple = qright.get(True)
            self.sources = rtuple.sources

            if (not (rtuple.data == "EOF")):

                while (not (rtuple.data == "EOF")):
                    # Create solution mapping.
                    data = {}
                    data.update(rtuple.data)
                    data.update(tuple.data)

                    # print("{}; {}".format(self.id_operator, data))
                    # Update ready and done vectors of solution mapping.
                    ready = rtuple.ready | tuple.ready
                    done = rtuple.done | tuple.done | pow(2, self.id_operator)
                    sources = list(set(rtuple.sources) | set(tuple.sources))

                    # Create tuple.
                    res = Tuple(data, ready, done, sources, self.id_operator)

                    # Send tuple to eddy operators.
                    self.qresults[self.eddy].put(res)

                    # Create and insert the record in the left_plan RJT table.
                    record = Record(rtuple, probeTS, time(), float("inf"))
                    if resource in rjttable:
                        other_rjttable.get(resource).updateRecords(record)
                        other_rjttable.get(resource).setRJTProbeTS(probeTS)
                    else:
                        tail = RJTTail(record, float("inf"))
                        other_rjttable[resource] = tail

                    rtuple = qright.get(True)

            else:
                # Build the empty tuple.
                rtuple = {}
                for att in self.right.vars:
                    rtuple.update({att: ''})

                # Produce the answer,
                rtuple.update(tuple.data)
                # Create tuple.
                sources = list(set(tuple.sources))
                done = tuple.done | pow(2, self.id_operator)
                ready = tuple.ready
                res = Tuple(rtuple, ready, done, sources, self.id_operator)
                self.qresults[self.eddy].put(res)

        return probeTS