示例#1
0
    def update(self):
        """Update local data

        This method downloads board data and save them in local file.
        When requesting http GET method, "If-Modified-Since" field is
        used to avoid unnecessary transfer.  "Accept-Encoding: gzip"
        is used too.

        """
        # Download data
        info = self._load_info()
        headers = {"Accept-Encoding": "gzip",
                   "If-Modified-Since": info.get("last-modified","")}
        response = http.get(self._host, "/%s/subject.txt"%self._path, headers)
        stat = response["status"]
        if stat==http.OK:
            # Save files
            d = os.path.dirname(self._data_file_path)
            if not os.path.exists(d):
                os.makedirs(d)
            file(self._data_file_path,"w").write(response["content"])
            info.update({"last-modified":response["last-modified"]})
            self._save_info(info)
        elif stat==http.NOT_MODIFIED:
            pass
        else:
            raise DownloadError, stat
示例#2
0
    def update(self):
        """Update local data

        This method downloads message data (.dat file) and store it
        locally.  Last-Modified and Range fields are used to minimize
        data transfer.  

        If the local file exists, the file size is set to http's
        "Range:" header, and "Last-Modified" value, stored in info
        file, is set to "If-Modified-Since" header. If the local file
        does not exists, these header fields are blank.

        After the header preparation, message data chunk is downloaded
        and saved locally.  If partial data are transfered, they are
        appended to the existing local data.  "Last-Modified" is save
        in info file.

        Also, this method revises summary so that Board instance is able
        to know about topic.

        DownloadError raises if failed.

        """
        # Set up headers
        info = self._load_info()
        headers = {}
        if os.path.exists(self._data_file_path):
            size = os.stat(self._data_file_path).st_size
            headers["Range"] = "%d-" % size
        headers["If-Modified-Since"] = info.get("last-modified","")
        # Download data
        r = http.get(self._host, self._data_url_path, headers)
        # Save data and info
        status = r["status"]
        mode = {http.OK:"wb", http.PARTIAL_CONTENT:"ab"}
        if status==http.OK or status==http.PARTIAL_CONTENT:
            # Create directory if necessary
            if not os.path.exists(os.path.dirname(self._data_file_path)):
                os.makedirs(os.path.dirname(self._data_file_path))
            # Save files
            file(self._data_file_path,mode[status]).write(r["content"])
            size = r["content"].count("\n")
            info.update({"last-modified": r.get("last-modified",""),
                         "local-size": size,
                         "remote-size":size})
            self._save_info(info)
            Summary(self).set(self, "size", size)
            # Update size
            self._local_size = size
            self._remote_size = size
        elif status==http.NOT_MODIFIED:
            pass
        else:
            raise DownloadError, str(status)
示例#3
0
文件: bbs.py 项目: wozozo/KittyWalk2
    def update(self):
        """Update BBS

        This method downloads the board tree, saves the data and build 
        internal data structure.
        """
        # Download
        raw_data = http.get(self._board_tree_url_host,
                            self._board_tree_url_path)
        # Convert
        xml = _bbstable2xml(j(raw_data["content"]))
        # Save
        dirname = os.path.dirname(self._filepath)
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        file(self._filepath,"w").write(xml.encode("utf8"))
        # Build
        self._categories = self._build_categories()