Exemple #1
0
    def on_commit(self, _prefix, interest: Interest, face, _filter_id, _filter):
        param = interest.applicationParameters.toBytes()
        if not isinstance(param, bytes):
            print("Malformed request")
            return
        param = param.split(b'\x00')
        if len(param) != 4:
            print("Malformed request")
            return
        repo, branch, dest_branch, commit_msg = map(bytes.decode, param)

        env = os.environ
        env['GIT_COMMITTER_NAME'] = 'GitSync'
        env['GIT_WORK_TREE'] = os.path.join(os.path.expanduser(MOUNT_PATH), repo, branch)
        env['GIT_DIR'] = os.path.join(env['GIT_WORK_TREE'], '.git')
        repo_uri = "ndn::" + Name(GIT_PREFIX).append(repo).toUri()

        # Commit (blocking)
        subprocess.call(['git', 'commit', '-a', '-m', commit_msg], env=env)
        # Push
        os.spawnlpe(os.P_NOWAIT, 'git', 'git', 'push', repo_uri, 'HEAD:refs/heads/' + dest_branch, env)

        # Respond with Data
        data = Data(interest.name)
        data.content = struct.pack("i", PUSH_RESPONSE_SUCCESS)
        data.metaInfo.freshnessPeriod = 1000
        face.putData(data)
    def onInterest(self, prefix, interest, face, interestFilterId, filter):
        """
        Append the next segment number to the prefix and send a new data packet.
        If the last packet is sent, then call self._onFinished().
        """
        maxSegment = 2
        if self._segment >= maxSegment:
            # We have already called self._onFinished().
            return

        dump("Got interest", interest.toUri())

        # Make and sign a Data packet for the segment.
        self._segment += 1
        data = Data(Name(prefix).appendSegment(self._segment))
        content = "Segment number " + repr(self._segment)
        data.content = content
        self._keyChain.sign(data, self._certificateName)

        face.putData(data)
        dump("Sent data packet", data.name.toUri())

        if self._segment >= maxSegment:
            # We sent the final data packet.
            self._onFinished()
    def onInterest(self, prefix, interest, face, interestFilterId, filter):
        """
        Append the next segment number to the prefix and send a new data packet.
        If the last packet is sent, then call self._onFinished().
        """
        maxSegment = 2
        if self._segment >= maxSegment:
            # We have already called self._onFinished().
            return

        dump("Got interest", interest.toUri())

        # Make and sign a Data packet for the segment.
        self._segment += 1
        data = Data(Name(prefix).appendSegment(self._segment))
        content = "Segment number " + repr(self._segment)
        data.content = content
        self._keyChain.sign(data, self._certificateName)

        face.putData(data)
        dump("Sent data packet", data.name.toUri())

        if self._segment >= maxSegment:
            # We sent the final data packet.
            self._onFinished()
Exemple #4
0
    def onInterest(self, prefix, interest, transport, registeredPrefixId):
        """
        Append the next segment number to the prefix and send a new data packet.
        If the last packet is sent, then set self._enabled[0] = False.
        """
        maxSegment = 2
        if self._segment >= maxSegment:
            return

        dump("Got interest", interest.toUri())

        # Make and sign a Data packet for the segment.
        self._segment += 1
        data = Data(Name(prefix).appendSegment(self._segment))
        content = "Segment number " + repr(self._segment)
        data.content = content
        self._keyChain.sign(data, self._certificateName)
        encodedData = data.wireEncode()

        transport.send(encodedData.toBuffer())
        dump("Sent data packet", data.name.toUri())

        if self._segment >= maxSegment:
            # We sent the final data packet, so stop.
            self._enabled[0] = False
Exemple #5
0
 def on_branchinfo_interest(self, _prefix, interest: Interest, face, _filter_id, _filter):
     name = interest.name
     print("ON BRANCH INFO INTEREST", name.toUri())
     branch = name[-1].toEscapedString()
     if branch not in self.branches:
         return
     data = Data(interest.name)
     data.content = pickle.dumps(self.branches[branch])
     data.metaInfo.freshnessPeriod = 1000
     face.putData(data)
Exemple #6
0
    def on_reflist_interest(self, _prefix, interest: Interest, face, _filter_id, _filter):
        result = '\n'.join("{} refs/heads/{}".format(info.head, name)
                           for name, info in self.branches.items())
        result = result + '\n'

        print("On reflist -> return:", result)

        data = Data(interest.name)
        data.content = result.encode("utf-8")
        data.metaInfo.freshnessPeriod = 1000
        face.putData(data)
Exemple #7
0
    def on_update(self, _prefix, interest: Interest, face, _filter_id,
                  _filter):
        # Decode Interest
        logging.info("OnClone: %s", interest.name.toUri())
        repo = interest.name[-2].toEscapedString()
        branch = interest.name[-1].toEscapedString()

        # Analyze commit
        commit = None
        if repo in self.repos:
            repo_obj = self.repos[repo]
            if branch in repo_obj.branches:
                commit = repo_obj.branches[branch].head
        if not commit:
            # We should fetch
            data = Data(interest.name)
            data.content = struct.pack("i", PUSH_RESPONSE_FAILURE)
            data.metaInfo.freshnessPeriod = 1000
            face.putData(data)
            return

        # Call git to mount (not efficient)
        alphabet = [
            '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C',
            'D', 'E', 'F'
        ]
        mount_path = os.path.join(os.path.expanduser(MOUNT_PATH), repo)
        os.makedirs(mount_path, exist_ok=True)
        mount_path = os.path.join(
            mount_path, "tmp-" + "".join(random.choices(alphabet, k=25)))
        repo_uri = "ndn::" + Name(GIT_PREFIX).append(repo).toUri()

        os.spawnlp(os.P_NOWAIT, 'git', 'git', 'clone', '--depth', '1',
                   '--branch', branch, repo_uri, mount_path)

        # Respond with Data
        data = Data(interest.name)
        data.content = struct.pack(
            "i", PUSH_RESPONSE_SUCCESS) + mount_path.encode()
        data.metaInfo.freshnessPeriod = 1000
        face.putData(data)
Exemple #8
0
    async def push(self, branch, commit, timeout, face, name):
        # TODO Check if new head is legal
        fetcher = self.fetch(commit)
        result = False

        async def checkout():
            nonlocal fetcher, result
            await fetcher.wait_until_finish()
            if not fetcher.success:
                return

            # TODO W-A-W conflict
            timestamp = await self.sync.publish_data(branch)
            self.branches[branch].timestamp = timestamp
            self.branches[branch].head = commit

            # Fix the database
            head_data_name = Name(self.repo_prefix).append("refs")
            head_data_name = head_data_name.append(branch).appendTimestamp(
                timestamp)
            head_data = Data(head_data_name)
            head_data.content = commit.encode("utf-8")
            # TODO Sign data
            self.branches[branch].head_data = head_data.wireEncode().toBytes()
            self.repo_db.put(branch, pickle.dumps(self.branches[branch]))
            self.branches[branch].head_data = b""
            result = True

        event_loop = asyncio.get_event_loop()
        response = None

        if branch not in self.branches:
            response = PUSH_RESPONSE_FAILURE

        if response is None:
            try:
                await asyncio.wait_for(fetcher.wait_until_finish(), timeout)
            except asyncio.TimeoutError:
                event_loop.create_task(checkout())
                response = PUSH_RESPONSE_PENDING

        if response is None:
            await asyncio.wait_for(checkout(), None)
            if result:
                response = PUSH_RESPONSE_SUCCESS
            else:
                response = PUSH_RESPONSE_FAILURE

        logging.info("Push Result: %s", response)
        data = Data(name)
        data.content = struct.pack("i", response)
        data.metaInfo.freshnessPeriod = 1000
        face.putData(data)
Exemple #9
0
    def on_unmount(self, _prefix, interest: Interest, face, _filter_id, _filter):
        logging.info("OnUnmount: %s", interest.name.toUri())
        repo = interest.name[-2].toEscapedString()
        branch = interest.name[-1].toEscapedString()
        dest_path = os.path.join(os.path.expanduser(MOUNT_PATH), repo, branch)

        shutil.rmtree(dest_path, ignore_errors=True)

        # Respond with Data
        data = Data(interest.name)
        data.content = struct.pack("i", PUSH_RESPONSE_SUCCESS)
        data.metaInfo.freshnessPeriod = 1000
        face.putData(data)
Exemple #10
0
    def on_mount(self, _prefix, interest: Interest, face, _filter_id, _filter):
        # TODO: Do a better job (mount, unmount, commit); support fetching and commit
        #       Recover from malformed request
        # Decode Interest
        logging.info("OnMount: %s", interest.name.toUri())
        repo = interest.name[-2].toEscapedString()
        branch = interest.name[-1].toEscapedString()

        # Analyze commit
        commit = None
        if repo in self.repos:
            repo_obj = self.repos[repo]
            if branch in repo_obj.branches:
                commit = repo_obj.branches[branch].head
        if not commit:
            # We should fetch
            data = Data(interest.name)
            data.content = struct.pack("i", PUSH_RESPONSE_FAILURE)
            data.metaInfo.freshnessPeriod = 1000
            face.putData(data)
            return

        # Call git to mount (not efficient)
        mount_path = os.path.join(os.path.expanduser(MOUNT_PATH), repo)
        os.makedirs(mount_path, exist_ok=True)
        mount_path = os.path.join(mount_path, commit)
        repo_uri = "ndn::" + Name(GIT_PREFIX).append(repo).toUri()

        os.spawnlp(os.P_NOWAIT, 'git', 'git', 'clone', '--depth', '1',
                   '--branch', branch, repo_uri, mount_path)

        # Respond with Data
        data = Data(interest.name)
        data.content = struct.pack(
            "i", PUSH_RESPONSE_SUCCESS) + mount_path.encode()
        data.metaInfo.freshnessPeriod = 1000
        face.putData(data)
Exemple #11
0
 def on_track_repo(self, _prefix, interest: Interest, face, _filter_id, _filter):
     logging.info("OnTrackRepo: %s", interest.name.toUri())
     if len(interest.name) < 2:
         return
     repo = interest.name[-1].toEscapedString()
     if repo in self.repos:
         response = PUSH_RESPONSE_FAILURE
     else:
         self.repos[repo] = Repo(self.objects_db, repo, self.face)
         self.repos_db.put(repo, b"")
         response = PUSH_RESPONSE_SUCCESS
     data = Data(interest.name)
     data.content = struct.pack("i", response)
     data.metaInfo.freshnessPeriod = 1000
     face.putData(data)
Exemple #12
0
 def on_create_branch(self, _prefix, interest: Interest, face, _filter_id, _filter):
     logging.info("OnCreateBranch: %s", interest.name.toUri())
     if len(interest.name) < 3:
         return
     repo = interest.name[-2].toEscapedString()
     branch = interest.name[-1].toEscapedString()
     if repo not in self.repos:
         response = PUSH_RESPONSE_FAILURE
     else:
         result = self.repos[repo].create_branch(branch, self.cmd_prefix.toUri())
         response = PUSH_RESPONSE_SUCCESS if result else PUSH_RESPONSE_FAILURE
     data = Data(interest.name)
     data.content = struct.pack("i", response)
     data.metaInfo.freshnessPeriod = 1000
     face.putData(data)
Exemple #13
0
    def on_mount(self, _prefix, interest: Interest, face, _filter_id, _filter):
        logging.info("OnMount: %s", interest.name.toUri())
        repo = interest.name[-2].toEscapedString()
        branch = interest.name[-1].toEscapedString()
        mount_path = os.path.join(os.path.expanduser(MOUNT_PATH), repo)
        os.makedirs(mount_path, exist_ok=True)
        repo_uri = "ndn::" + Name(GIT_PREFIX).append(repo).toUri()

        os.spawnlp(os.P_NOWAIT, 'git', 'git', 'clone', '--single-branch',
                   '--branch', branch, repo_uri, os.path.join(mount_path, branch))

        # Respond with Data
        data = Data(interest.name)
        data.content = struct.pack("i", PUSH_RESPONSE_SUCCESS)
        data.metaInfo.freshnessPeriod = 1000
        face.putData(data)
Exemple #14
0
        async def checkout():
            nonlocal fetcher, result
            await fetcher.wait_until_finish()
            if not fetcher.success:
                return

            # TODO W-A-W conflict
            timestamp = await self.sync.publish_data(branch)

            head_data_name = Name(self.repo_prefix).append("refs")
            head_data_name = head_data_name.append(branch).appendTimestamp(timestamp)
            head_data = Data(head_data_name)
            head_data.content = commit.encode("utf-8")
            # TODO Sign data

            self.update_branch(branch, timestamp, commit, head_data.wireEncode().toBytes())
            result = True
Exemple #15
0
    def on_interest(self, _prefix, interest, face, _filter_id, _filter):
        if len(interest.name) < 4:
            return
        if interest.name[-1].isSequenceNumber():
            _ = interest.name[-1].toSequenceNumber()
            hash_name = interest.name[-2].toEscapedString()
        else:
            hash_name = interest.name[-1].toEscapedString()

        # TODO: Segmentation
        logging.info("OnInterest: %s", interest.name.toUri())
        if self.storage.exists(hash_name):
            data = Data(interest.name)
            data.content = self.storage.get(hash_name)
            data.metaInfo.freshnessPeriod = 60000
            face.putData(data)
        else:
            logging.info("Not exist: %s", hash_name)
Exemple #16
0
    def update_branch(self, branch: str, timestamp: int, commit: str, head_data: bytes):
        # Update database
        self.branches[branch].timestamp = timestamp
        self.branches[branch].head = commit
        self.branches[branch].head_data = head_data
        self.repo_db.put(branch, pickle.dumps(self.branches[branch]))
        # Release Data in memory
        self.branches[branch].head_data = b""

        # Send notification Data
        notif = Data(Name(LOCAL_CMD_PREFIX)
                     .append("notif")
                     .append(self.repo_name)
                     .append(branch)
                     .appendTimestamp(Sync.timestamp()))
        notif.metaInfo.freshnessPeriod = 10
        notif.content = commit.encode()
        self.face.putData(notif)
Exemple #17
0
    def onInterest(self, prefix, interest, face, interestFilterId, filter):
        """
        Create and send a Data packet with the interest name.
        If the last packet is sent, then set self._enabled[0] = False.
        """
        dump("Got interest in produce segments object:", interest.toUri())

        # Make and sign a Data packet with the interest name.
        data = Data(interest.name)
        content = "Data packet " + interest.name.toUri()
        data.content = content
        self._keyChain.sign(data, self._certificateName)

        face.putData(data)
        dump("Sent data packet", data.name.toUri())

        self._nSegmentsSent += 1
        if self._nSegmentsSent >= (self._endBlockId - self._startBlockId) + 1:
            # We sent the final segment.
            self._onFinished()
Exemple #18
0
    def onInterest(self, prefix, interest, face, interestFilterId, filter):
        """
        Create and send a Data packet with the interest name.
        If the last packet is sent, then set self._enabled[0] = False.
        """
        dump("Got interest", interest.toUri())

        # Make and sign a Data packet with the interest name.
        data = Data(interest.name)
        content = "Data packet " + interest.name.toUri()
        data.content = content
        self._keyChain.sign(data, self._certificateName)

        face.putData(data)
        dump("Sent data packet", data.name.toUri())

        self._nSegmentsSent += 1
        if self._nSegmentsSent >= (self._endBlockId - self._startBlockId) + 1:
            # We sent the final segment.
            self._onFinished()
Exemple #19
0
    def on_push(self, _prefix, interest: Interest, face, _filter_id, _filter):
        logging.info("OnPush: %s", interest.name.toUri())
        if len(interest.name) < 3:
            return
        repo = interest.name[-3].toEscapedString()
        branch = interest.name[-2].toEscapedString()
        if repo not in self.repos:
            logging.info("Repo %s doesn't exist", repo)
            response = PUSH_RESPONSE_FAILURE
            data = Data(interest.name)
            data.content = struct.pack("i", response)
            face.putData(data)
        else:
            commit = interest.applicationParameters.toBytes().decode("utf-8")
            timeout = interest.interestLifetimeMilliseconds / 1000.0 / 2.0

            logging.info("Arguments %s %s %s %d", repo, branch, commit, timeout)

            event_loop = asyncio.get_event_loop()
            event_loop.create_task(self.repos[repo].push(branch, commit, timeout, face, interest.name))
Exemple #20
0
        async def checkout():
            nonlocal fetcher, result
            await fetcher.wait_until_finish()
            if not fetcher.success:
                return

            # TODO W-A-W conflict
            timestamp = await self.sync.publish_data(branch)
            self.branches[branch].timestamp = timestamp
            self.branches[branch].head = commit

            # Fix the database
            head_data_name = Name(self.repo_prefix).append("refs")
            head_data_name = head_data_name.append(branch).appendTimestamp(
                timestamp)
            head_data = Data(head_data_name)
            head_data.content = commit.encode("utf-8")
            # TODO Sign data
            self.branches[branch].head_data = head_data.wireEncode().toBytes()
            self.repo_db.put(branch, pickle.dumps(self.branches[branch]))
            self.branches[branch].head_data = b""
            result = True
Exemple #21
0
    def on_result_interest(self, _prefix, interest, face, _interest_filter_id, _filter_obj):
        # type: (Name, Interest, Face, int, InterestFilter) -> bool
        prefix = Name(SERVER_PREFIX).append(RESULT_PREFIX)
        if not prefix.isPrefixOf(interest.name):
            # Wrong prefix
            return False

        data_name = interest.name[prefix.size():]
        logging.info("On result interest: %s", data_name.toUri())
        # key, stat = self._result_set_prefix_match(data_name)
        status = self.load_status(data_name)
        if status is None:
            # No such request
            self.nodata_reply(interest.name, RET_NO_REQUEST)
            return True

        if data_name[-1].isSegment():
            # Segment no suffix
            seg_no = data_name[-1].toSegment()
            result = self.storage.get(data_name.getPrefix(-1))
        elif data_name[-1] == Name("_meta")[0]:
            # MetaInfo suffix
            seg_no = -1
            result = self.storage.get(data_name.getPrefix(-1))
        else:
            # No suffix
            seg_no = None
            result = self.storage.get(data_name)

        if result is not None:
            # There are data
            segment_cnt = (len(result) + self.segment_size - 1) // self.segment_size
            # Note: I don't understand why namespace keep all data in memory
            metainfo = MetaInfo()
            # metainfo.setFinalBlockId(segment_cnt - 1) # WHY this doesn't work?
            metainfo.setFinalBlockId(Name().appendSegment(segment_cnt - 1)[0])
            if segment_cnt > 1 and seg_no is None:
                # Fetch segmented data with no suffix will get only first segment
                seg_no = 0
                data_name.appendSequenceNumber(seg_no)

            data = Data(Name(prefix).append(data_name))
            data.setMetaInfo(metainfo)
            if seg_no == -1:
                # _meta
                data.content = self.storage.get(data_name)
            else:
                # data
                if segment_cnt > 1:
                    # Segmented
                    if seg_no < segment_cnt:
                        start_offset = seg_no * self.segment_size
                        end_offset = start_offset + self.segment_size
                        data.content = Blob(bytearray(result[start_offset:end_offset]))
                    else:
                        data.content = None
                else:
                    # No segmentation
                    data.content = Blob(bytearray(result))

            self.keychain.sign(data)
            face.putData(data)
            return True
        else:
            # Data are not ready
            if status.status == STATUS_NO_INPUT:
                self.nodata_reply(interest.name, RET_NO_INPUT)
            elif status.status == STATUS_FAILED:
                self.nodata_reply(interest.name, RET_EXECUTION_FAILED)
            else:
                self.nodata_reply(interest.name, RET_RETRY_AFTER, status.estimated_time - Common.getNowMilliseconds())
            return True