def retrieve_2ch_boards(url="http://menu.2ch.net/bbsmenu.html", user=None):
my_user = user if user else make_anonymous_user()
response = my_user.urlopen(url, gzip=False)
if response.code == 200:
html_string = unicode(response.read(), "Shift_JIS", "ignore")
return parse_2ch_boards(html_string)
else:
message = "HTTP status is invalid: %i" + str(response.code)
raise HttpStatusError(message, response)
def retrieve_board(board_url, user=None):
my_user = user if user else make_anonymous_user()
subject_url = make_subject_url(board_url)
response = my_user.urlopen(subject_url, gzip=False)
if response.code == 200:
retrieved_string = unicode(response.read(), "Shift_JIS", "ignore")
print type(retrieved_string)
return parse_board(retrieved_string)
else:
message = "HTTP status is invalid: " + str(response.code)
raise HttpStatusError(message, response)
def retrieve_thread(board_url, dat_name, user=None):
my_user = user if user else make_anonymous_user()
target_url = make_dat_url(board_url, dat_name)
response = my_user.urlopen(target_url, gzip=True)
if response.code == 200:
zipped_IO = StringIO.StringIO(response.read())
unzipped_string = gzip.GzipFile(fileobj=zipped_IO).read()
dat_string = unicode(unzipped_string, "Shift_JIS", "replace")
return Thread(dat_string)
else:
message = "HTTP status is invalid: " + str(response.code)
raise HttpStatusError(message, response)
