def update(self):
"""Update local data
This method downloads board data and save them in local file.
When requesting http GET method, "If-Modified-Since" field is
used to avoid unnecessary transfer. "Accept-Encoding: gzip"
is used too.
"""
# Download data
info = self._load_info()
headers = {"Accept-Encoding": "gzip",
"If-Modified-Since": info.get("last-modified","")}
response = http.get(self._host, "/%s/subject.txt"%self._path, headers)
stat = response["status"]
if stat==http.OK:
# Save files
d = os.path.dirname(self._data_file_path)
if not os.path.exists(d):
os.makedirs(d)
file(self._data_file_path,"w").write(response["content"])
info.update({"last-modified":response["last-modified"]})
self._save_info(info)
elif stat==http.NOT_MODIFIED:
pass
else:
raise DownloadError, stat
def update(self):
"""Update local data
This method downloads message data (.dat file) and store it
locally. Last-Modified and Range fields are used to minimize
data transfer.
If the local file exists, the file size is set to http's
"Range:" header, and "Last-Modified" value, stored in info
file, is set to "If-Modified-Since" header. If the local file
does not exists, these header fields are blank.
After the header preparation, message data chunk is downloaded
and saved locally. If partial data are transfered, they are
appended to the existing local data. "Last-Modified" is save
in info file.
Also, this method revises summary so that Board instance is able
to know about topic.
DownloadError raises if failed.
"""
# Set up headers
info = self._load_info()
headers = {}
if os.path.exists(self._data_file_path):
size = os.stat(self._data_file_path).st_size
headers["Range"] = "%d-" % size
headers["If-Modified-Since"] = info.get("last-modified","")
# Download data
r = http.get(self._host, self._data_url_path, headers)
# Save data and info
status = r["status"]
mode = {http.OK:"wb", http.PARTIAL_CONTENT:"ab"}
if status==http.OK or status==http.PARTIAL_CONTENT:
# Create directory if necessary
if not os.path.exists(os.path.dirname(self._data_file_path)):
os.makedirs(os.path.dirname(self._data_file_path))
# Save files
file(self._data_file_path,mode[status]).write(r["content"])
size = r["content"].count("\n")
info.update({"last-modified": r.get("last-modified",""),
"local-size": size,
"remote-size":size})
self._save_info(info)
Summary(self).set(self, "size", size)
# Update size
self._local_size = size
self._remote_size = size
elif status==http.NOT_MODIFIED:
pass
else:
raise DownloadError, str(status)
def update(self):
"""Update BBS
This method downloads the board tree, saves the data and build
internal data structure.
"""
# Download
raw_data = http.get(self._board_tree_url_host,
self._board_tree_url_path)
# Convert
xml = _bbstable2xml(j(raw_data["content"]))
# Save
dirname = os.path.dirname(self._filepath)
if not os.path.exists(dirname):
os.makedirs(dirname)
file(self._filepath,"w").write(xml.encode("utf8"))
# Build
self._categories = self._build_categories()