Exemple #1
0
    def subscribeRequest(self, request, context):
        print "\nFRONTEND : new client subscriber ", request.client_ip, " for topic", request.topic
        subType = ""
        if subscribers.find({"topic": request.topic}).count() > 0:
            print "FRONTEND : Not the first client for topic ", request.topic
            subType = "old"
        else:
            print "FRONTEND : First client for topic ", request.topic
            subType = "new"
        newSubscribers.insert_one({
            "topic": request.topic,
            "ip": request.client_ip
        })
        subscribers.insert_one({
            "topic": request.topic,
            "ip": request.client_ip
        })
        if subscribers.find({"topic": request.topic}).count() > THRESHOLD:
            channel = grpc.insecure_channel(CENTRAL_SERVER_IP)
            stub = pr_pb2_grpc.PublishTopicStub(channel)
            response = stub.replicaRequest(
                pr_pb2.topicSubscribe(topic=request.topic,
                                      client_ip=str(SELF_IP) + ":" + port))

        channel = grpc.insecure_channel(CENTRAL_SERVER_IP)
        stub = pr_pb2_grpc.PublishTopicStub(channel)
        response = stub.subscribeRequestCentral(
            pr_pb2.topicDataType(topic=request.topic,
                                 client_ip=str(SELF_IP) + ":" + str(port),
                                 type=subType))
        return pr_pb2.acknowledge(ack="temporary acknowledge")
Exemple #2
0
 def unsubscribeRequest(self, request_iterator, context):
     topicList = []
     client_ip = ""
     for request in request_iterator:
         print "FRONTEND : Unsubscribe request from client", request.client_ip, " for topic", request.topic
         client_ip = request.client_ip
         topicList.append(request.topic)
     for topic in topicList:
         subscribers.delete_one({"topic": topic, "ip": client_ip})
         count = subscribers.find({"topic": topic}).count()
         if count <= THRESHOLD and count > 0:
             channel = grpc.insecure_channel(CENTRAL_SERVER_IP)
             stub = pr_pb2_grpc.PublishTopicStub(channel)
             response = stub.deReplicaRequest(
                 pr_pb2.topicSubscribe(topic=topic,
                                       client_ip=str(SELF_IP) + ":" + port))
             if response.ack == "DONE":
                 dataDump.delete_many({"topic": topic})
                 print "TOPIC_SERVER : Dereplicated for topic :", topic
         else:
             channel = grpc.insecure_channel(CENTRAL_SERVER_IP)
             stub = pr_pb2_grpc.PublishTopicStub(channel)
             response = stub.unsubscribeRequestCentral(
                 pr_pb2.topicSubscribe(topic=topic,
                                       client_ip=str(SELF_IP) + ":" + port))
     return pr_pb2.acknowledge(ack="done")
Exemple #3
0
    def replicaRequest(self, request, context):
        document = twoLevelDict.find_one({"topic":request.topic,"subscriber":"NULL"})
        allotedServer = document["publisher"]
        if twoLevelDict.find({"topic":request.topic,"publisher":request.client_ip}).count() > 0 :
            return pr_pb2.acknowledge(ack="Requesting front end server already a replica for "+request.topic)
        if twoLevelDict.find({"topic":request.topic,"subscriber":request.client_ip}).count() > 0 :
            if IS_MASTER:
                response = two_phase_init(pr_pb2.commit_req_data(action="remove",level="3",data_1 = request.topic,data_2="", data_3 = request.client_ip, filename = "twoLevelDict",function_name="replicaRequest"))
                if response == "ERROR" :
                    twoLevelDict.delete_one({"topic":request.topic,"subscriber":request.client_ip})
            else :
                twoLevelDict.delete_one({"topic":request.topic,"subscriber":request.client_ip})
            # twoLevelDict.delete_one({"topic":request.topic,"subscriber":request.client_ip})

        if IS_MASTER:
            response = two_phase_init(pr_pb2.commit_req_data(action="insert",level="3",data_1 = request.topic,data_2=request.client_ip, data_3 = "NULL", filename = "twoLevelDict",function_name="replicaRequest"))
            if response == "ERROR" :
                twoLevelDict.insert_one({"topic":request.topic,"publisher":request.client_ip,"subscriber":"NULL"})
        else :
            twoLevelDict.insert_one({"topic":request.topic,"publisher":request.client_ip,"subscriber":"NULL"})
        # twoLevelDict.insert_one({"topic":request.topic,"publisher":request.client_ip,"subscriber":"NULL"})
        if IS_MASTER:
            response = two_phase_init(pr_pb2.commit_req_data(action="insert",level="3",data_1 = request.topic,data_2=request.client_ip, data_3 = request.client_ip, filename = "twoLevelDict",function_name="replicaRequest"))
            if response == "ERROR" :
                twoLevelDict.insert_one({"topic":request.topic,"publisher":request.client_ip,"subscriber":request.client_ip})
        else :
            twoLevelDict.insert_one({"topic":request.topic,"publisher":request.client_ip,"subscriber":request.client_ip})
        # twoLevelDict.insert_one({"topic":request.topic,"publisher":request.client_ip,"subscriber":request.client_ip})
        channel = grpc.insecure_channel(allotedServer)
        stub = pr_pb2_grpc.PublishTopicStub(channel)
        response = stub.sendBackupRequestReplica(pr_pb2.topicSubscribe(topic=request.topic, client_ip=request.client_ip))
        print "Replication for topic server : "+request.client_ip+" started..."
        return pr_pb2.acknowledge(ack="Requesting front end server "+request.client_ip+" made a replica of topic(backup sent) "+request.topic)
Exemple #4
0
    def subscribeRequestCentral(self, request, context):
        print "Subscribe request from access point",request.client_ip," for topic",request.topic," of type :",request.type
        allotedServer = ""
        if request.type == "new" :
            if twoLevelDict.find({"topic":request.topic,"publisher":request.client_ip}).count() > 0:
                allotedServer = request.client_ip 
            else :
                l = sys.maxsize
                tempIp = ""
                cursor = twoLevelDict.find({"topic":request.topic,"subscriber":"NULL"})
                for document in cursor :
                    ip = document["publisher"]
                    if twoLevelDict.find({"topic":request.topic,"publisher":ip}).count() < l :
                        l=twoLevelDict.find({"topic":request.topic,"publisher":ip}).count()
                        tempIp = ip
                allotedServer = tempIp
            if IS_MASTER:
                response = two_phase_init(pr_pb2.commit_req_data(action="insert",level="3",data_1 = request.topic,data_2=allotedServer, data_3 = request.client_ip, filename = "twoLevelDict",function_name="subscribeRequestCentral"))
                if response == "ERROR" :
                    twoLevelDict.insert_one({"topic":request.topic,"publisher":allotedServer,"subscriber":request.client_ip})
            else :
                twoLevelDict.insert_one({"topic":request.topic,"publisher":allotedServer,"subscriber":request.client_ip})
            # twoLevelDict.insert_one({"topic":request.topic,"publisher":allotedServer,"subscriber":request.client_ip})

        else :
            document = twoLevelDict.find_one({"topic":request.topic,"subscriber":request.client_ip})
            allotedServer = document["publisher"]

        channel = grpc.insecure_channel(allotedServer)
        stub = pr_pb2_grpc.PublishTopicStub(channel)
        response = stub.sendBackupRequest(pr_pb2.topicSubscribe(topic=request.topic, client_ip=request.client_ip))
        print response.ack
        return pr_pb2.acknowledge(ack="temporary acknowledge from central server")
Exemple #5
0
def publishData(lst):
    request = lst[0]
    accesspoint = lst[1]
    channel = grpc.insecure_channel(accesspoint)
    stub = pr_pb2_grpc.PublishTopicStub(channel)
    response = stub.publish(
        pr_pb2.topicData(topic=request.topic, data=request.data))
    return response.ack
Exemple #6
0
def get_front_ip():
    channel = grpc.insecure_channel(CENTRAL_SERVER_IP)
    stub = pr_pb2_grpc.PublishTopicStub(channel)
    response = stub.getFrontIp(pr_pb2.empty())
    print("Frontend server Ip alloted: " + response.ip)
    if response.ip == "NONE":
        print "No frontend servers active ...extiting..."
        exit()
    return response.ip
Exemple #7
0
def subscribe_topic(topic, self_ip):
    if subscribedTopics.find({"topic": topic}).count() > 0:
        print "Already subscribed to the topic :", topic
    else:
        channel = grpc.insecure_channel(ACCESS_POINT)
        stub = pr_pb2_grpc.PublishTopicStub(channel)
        response = stub.subscribeRequest(
            pr_pb2.topicSubscribe(topic=topic, client_ip=self_ip))
        subscribedTopics.insert_one({"topic": topic})
Exemple #8
0
 def sendBackupRequest(self, request, context):
     cursor = dataDump.find({"topic": request.topic})
     lst = []
     for document in cursor:
         lst.append(document["data"])
     channel = grpc.insecure_channel(request.client_ip)
     stub = pr_pb2_grpc.PublishTopicStub(channel)
     response = stub.sendBackup(generateBackup(request.topic, lst))
     return pr_pb2.acknowledge(ack="data send to : " + request.client_ip +
                               " complete...")
Exemple #9
0
def forwardBackupToClient(lst):
    requestList = lst[0]
    client_ip = lst[1]
    lst = []
    temp_topic = ""
    for request in requestList:
        lst.append(request.data)
        temp_topic = request.topic
    channel = grpc.insecure_channel(client_ip)
    stub = pr_pb2_grpc.PublishTopicStub(channel)
    response = stub.forwardBackup(generateBackup(temp_topic, lst))
Exemple #10
0
 def sendBackupRequestReplica(self, request, context):
     print "TOPIC_SERVER : Sending data backup for topic : " + request.topic + " to the new replica : " + request.client_ip
     cursor = dataDump.find({"topic": request.topic})
     lst = []
     for document in cursor:
         lst.append(document["data"])
     channel = grpc.insecure_channel(request.client_ip)
     stub = pr_pb2_grpc.PublishTopicStub(channel)
     response = stub.sendBackupReplica(generateBackup(request.topic, lst))
     return pr_pb2.acknowledge(ack="data send to : " + request.client_ip +
                               " replica complete...")
Exemple #11
0
 def publishRequest(self, request, context):
     print "FRONTEND : Data received for topic : " + request.topic
     channel = grpc.insecure_channel(CENTRAL_SERVER_IP)
     stub = pr_pb2_grpc.PublishTopicStub(channel)
     responses = stub.giveIps(pr_pb2.topic(topic=request.topic))
     returned_ips = []
     for response in responses:
         print("FRONTEND : Data to be sent to topic server IP: " +
               response.ip + " for topic: " + request.topic)
         returned_ips.append(response.ip)
     lst = []
     pool = ThreadPool(len(returned_ips))
     for returned_ip in returned_ips:
         lst.append([request, returned_ip])
     results = pool.map(publishData, lst)
     return pr_pb2.acknowledge(ack="Published in " + str(len(results)) +
                               " topic servers")
Exemple #12
0
def two_phase_init(request):
    while(IS_LOCKED == True):
        pass
    channel = grpc.insecure_channel(backupCentralServer)
    stub = pr_pb2_grpc.PublishTopicStub(channel)
    retries = 2
    while (retries>0) :
        try:
            response = stub.commit_request(request)
            if response.ack=="OK":
                logging.info("%s:%s:COMPLETE",str(datetime.now()),request.filename)
                return "COMPLETE"
            else :
                return "ERROR"

        except Exception as e:
            retries -= 1
            if retries==0:
                logging.info("%s:%s:Backup down, performing transaction...",str(datetime.now()),request.filename)
                print "Backup down...performing transaction"
                return "ERROR"
Exemple #13
0
 def publish(self, request, context):
     print "TOPIC_SERVER : Data received for topic : " + request.topic
     dataDump.insert_one({"topic": request.topic, "data": request.data})
     channel = grpc.insecure_channel(CENTRAL_SERVER_IP)
     stub = pr_pb2_grpc.PublishTopicStub(channel)
     responses = stub.giveSubscriberIps(
         pr_pb2.topicSubscribe(topic=request.topic,
                               client_ip=str(SELF_IP) + ":" + port))
     ipList = []
     for response in responses:
         ipList.append(response.ip)
         print("TOPIC_SERVER : frontend subscriber IP received: " +
               response.ip + " for topic: " + request.topic)
     if ipList[0] == "none":
         return pr_pb2.acknowledge(ack="No subscribers for this replica")
     pool = ThreadPool(len(ipList))
     lst = []
     for client_ip in ipList:
         lst.append([request, client_ip])
     results = pool.map(sendToFrontend, lst)
     return pr_pb2.acknowledge(ack="Data send to clients complete")
Exemple #14
0
def push_topic(topic, data):
    channel = grpc.insecure_channel(ACCESS_POINT)
    stub = pr_pb2_grpc.PublishTopicStub(channel)
    response = stub.publishRequest(pr_pb2.topicData(topic=topic, data=data))
    print("Ack received: " + response.ack)
Exemple #15
0
        print "Type 1 for publish\nType 2 for subscribe\nType 3 for unsubscribe\nType 4 for exit"
        response = raw_input()

        if response == "1":
            print "Enter topic"
            topic = raw_input()

            print "Enter data"
            data = raw_input()

            push_topic(topic, data)

        elif response == "2":
            channel = grpc.insecure_channel(CENTRAL_SERVER_IP)
            stub = pr_pb2_grpc.PublishTopicStub(channel)
            responses = stub.querryTopics(pr_pb2.empty())
            topicList = []
            i = 0
            for response in responses:
                i += 1
                topicList.append(response.topic)
            cursor = subscribedTopics.find({})
            subscribedTopicsList = []
            for document in cursor:
                subscribedTopicsList.append(document["topic"])
            newTopicList = list(set(topicList) - set(subscribedTopicsList))
            i = 0
            for topic in newTopicList:
                print i, ": ", topic
                i += 1
Exemple #16
0
    def commit_request(self,request,context):
        if request.filename == "twoLevelDict":        
            if request.action == "remove" :
                if request.level == "3" : 
                    documents = twoLevelDict.find({"topic":request.data_1,"subscriber":request.data_3})
                    for document in documents:
                        pending_twoLevelDict.insert_one({"topic":document["topic"],"publisher":document["publisher"],"subscriber":document["subscriber"],"action":"remove"})
                    twoLevelDict.delete_one({"topic":request.data_1,"subscriber":request.data_3})
                    logging.info("%s:%s:REMOVE 3 %s %s %s",str(datetime.now()),request.filename,request.data_1,"NIL",request.data_3)
                    
                elif request.level == "2" :
                    documents = twoLevelDict.find({"topic":request.data_1,"publisher":request.data_2})
                    for document in documents:
                        pending_twoLevelDict.insert_one({"topic":document["topic"],"publisher":document["publisher"],"subscriber":document["subscriber"],"action":"remove"})
                    twoLevelDict.delete_many({"topic":request.data_1,"publisher":request.data_2})
                    logging.info("%s:%s:REMOVE 2 %s %s %s",str(datetime.now()),request.filename,request.data_1,request.data_2,"NIL")
                    
            elif request.action == "insert":
                if request.level == "3" :
                    twoLevelDict.insert_one({"topic":request.data_1,"publisher":request.data_2,"subscriber":request.data_3}) 
                    documents = twoLevelDict.find({"topic":request.data_1,"publisher":request.data_2,"subscriber":request.data_3})
                    for document in documents:
                        pending_twoLevelDict.insert_one({"topic":document["topic"],"publisher":document["publisher"],"subscriber":document["subscriber"],"action":"insert"})
                    logging.info("%s:%s:INSERT 3 %s %s %s",str(datetime.now()),request.filename,request.data_1,request.data_2,request.data_3)
                   
        elif request.filename == "frontends" :
            if request.action == "remove" :
                if request.level == "2" : 
                    if request.data_1 == "ip" :
                        pending_frontends.insert_one({"type":request.data_1,request.data_1:request.data_2,"action":"remove"})
                        frontends.delete_one({"type":request.data_1,request.data_1:request.data_2})
                        logging.info("%s:%s:REMOVE 2 %s %s %s",str(datetime.now()),request.filename,request.data_1,request.data_2,request.data_3)
                    else :
                        pending_frontends.insert_one({"type":request.data_1,request.data_1:int(request.data_2),"action":"remove"})
                        frontends.delete_one({"type":request.data_1,request.data_1:int(request.data_2)})
                        logging.info("%s:%s:REMOVE 2 %s %s %s",str(datetime.now()),request.filename,request.data_1,request.data_2,request.data_3)

            elif request.action == "insert" :
                if request.level == "2" : 
                    if request.data_1 == "ip" :
                        pending_frontends.insert_one({"type":request.data_1,request.data_1:request.data_2,"action":"insert"})
                        frontends.insert_one({"type":request.data_1,request.data_1:request.data_2})
                        logging.info("%s:%s:INSERT 2 %s %s %s",str(datetime.now()),request.filename,request.data_1,request.data_2,request.data_3)
                    else :
                        pending_frontends.insert_one({"type":request.data_1,request.data_1:int(request.data_2),"action":"insert"})
                        frontends.insert_one({"type":request.data_1,request.data_1:int(request.data_2)})
                        logging.info("%s:%s:INSERT 2 %s %s %s",str(datetime.now()),request.filename,request.data_1,request.data_2,request.data_3)
        
        channel = grpc.insecure_channel(centralServer)
        stub = pr_pb2_grpc.PublishTopicStub(channel)

        TIMEOUT = 3
        print str(TIMEOUT),"seconds to kill central server..."
        time.sleep(TIMEOUT)

        retries = 0
        while(True) :
            try :
                response = stub.commit_phase_two(request)
                if response.ack == "COMMIT":
                    pending_frontends.drop()
                    pending_twoLevelDict.drop()
                    logging.info("%s:%s:COMMIT",str(datetime.now()),request.filename)
                    return pr_pb2.acknowledge(ack="OK")
                else :
                    return pr_pb2.acknowledge(ack="ERROR")
            except :
                retries += 1
                if retries > MAX_RETRIES :
                    print "Central down..."
                    roll_back(request)
                    logging.info("%s:%s:ROLLBACK",str(datetime.now()),request.filename)
                    return pr_pb2.acknowledge(ack="ERROR")
Exemple #17
0
def register_ip():
    channel = grpc.insecure_channel(CENTRAL_SERVER_IP)
    stub = pr_pb2_grpc.PublishTopicStub(channel)
    response = stub.registerIp(pr_pb2.ips(ip=str(SELF_IP) + ":" + str(port)))
    print "Ip added to central server..."
Exemple #18
0
def sendToFrontend(lst):
    request = lst[0]
    client_ip = lst[1]
    channel = grpc.insecure_channel(client_ip)
    stub = pr_pb2_grpc.PublishTopicStub(channel)
    response = stub.sendData(request)