def retrieve_2ch_boards(url="http://menu.2ch.net/bbsmenu.html", user=None): my_user = user if user else make_anonymous_user() response = my_user.urlopen(url, gzip=False) if response.code == 200: html_string = unicode(response.read(), "Shift_JIS", "ignore") return parse_2ch_boards(html_string) else: message = "HTTP status is invalid: %i" + str(response.code) raise HttpStatusError(message, response)
def retrieve_board(board_url, user=None): my_user = user if user else make_anonymous_user() subject_url = make_subject_url(board_url) response = my_user.urlopen(subject_url, gzip=False) if response.code == 200: retrieved_string = unicode(response.read(), "Shift_JIS", "ignore") print type(retrieved_string) return parse_board(retrieved_string) else: message = "HTTP status is invalid: " + str(response.code) raise HttpStatusError(message, response)
def retrieve_thread(board_url, dat_name, user=None): my_user = user if user else make_anonymous_user() target_url = make_dat_url(board_url, dat_name) response = my_user.urlopen(target_url, gzip=True) if response.code == 200: zipped_IO = StringIO.StringIO(response.read()) unzipped_string = gzip.GzipFile(fileobj=zipped_IO).read() dat_string = unicode(unzipped_string, "Shift_JIS", "replace") return Thread(dat_string) else: message = "HTTP status is invalid: " + str(response.code) raise HttpStatusError(message, response)