def generate_large_example_csv(page_start=1, page_end=20, boards=['pol', 'x']):
results = {}
for i in range(page_start, page_end):
for board in boards:
results.update(**httpGET_json(gen_index_api_url(board, i)))
print("{}th page...".format(i))
postList = FourPlebsAPI_Post.from_post_json(results)
CSVPostWriter.write_posts_to_csv(postList, 'out/post-output-large.csv',
ALL_CONTENT_FLAGGERS)
def generate_csv():
boardsDesc = "The boards on 4chan you wish to gather from."
boards = unpack_http_get_list(request.args.get('boards', None))
boards = parameter_blacklist(boards, 'boards', boardsDesc)
boards = parameters_must_be_inside_list(boards, BOARDS_4PLEBS, boardsDesc)
print(boards)
flaggerDesc = "The names of content flaggers to use. See {} for supported names.".format(
url_for('content_flaggers'))
flaggers = unpack_http_get_list(request.args.get('flaggers', None))
flaggers = parameter_blacklist(flaggers, 'flaggers', flaggerDesc)
flaggers = parameters_must_be_inside_list(flaggers,
content_flagger_names(),
flaggerDesc)
print(flaggers)
start_page = request.args.get('start_page', None)
start_page = parameter_must_be_numeric(
start_page, 'start_page',
"The page of the imageboard's board to start gathering from.")
print(start_page)
stop_page = request.args.get('stop_page', None)
stop_page = parameter_must_be_numeric(
stop_page, 'stop_page',
"The page of the imageboard's board to finish gathering from.")
print(stop_page)
stringInputStream = io.StringIO()
posts = gather_range_with_boards(start=start_page,
end=stop_page,
boards=boards)
CSVPostWriter.write_posts_to_stream(
threads=posts,
stream=stringInputStream,
content_flaggers=content_flagger_names_to_ContentFlaggers(flaggers))
# TODO: use their content flagger selections!
output = make_response(stringInputStream.getvalue())
print('wow10:', output)
output.headers["Content-Disposition"] = "attachment; filename=export.csv"
output.headers["Content-type"] = "text/csv"
output.headers["charset"] = 'utf-8'
return output
def _generate_csv_string_4plebs(boards: str, flaggers: str, start_page: str,
stop_page: str) -> str:
boardsDesc = "The boards on 4chan you wish to gather from."
boards = unpack_http_get_list(boards)
boards = parameter_blacklist(boards, 'boards', boardsDesc)
boards = parameters_must_be_inside_list(boards, BOARDS_4PLEBS, boardsDesc)
print(boards)
flaggerDesc = "The names of content flaggers to use. See {} for supported names.".format(
url_for('content_flaggers'))
flaggers = unpack_http_get_list(flaggers)
flaggers = parameter_blacklist(flaggers, 'flaggers', flaggerDesc)
flaggers = parameters_must_be_inside_list(flaggers,
content_flagger_names(),
flaggerDesc)
print(flaggers)
start_page = parameter_must_be_numeric(
start_page, 'start_page',
"The page of the imageboard's board to start gathering from.")
print(start_page)
stop_page = parameter_must_be_numeric(
stop_page, 'stop_page',
"The page of the imageboard's board to finish gathering from.")
print(stop_page)
stringInputStream = io.StringIO()
posts = gather_range_with_boards(start=start_page,
end=stop_page,
boards=boards)
CSVPostWriter.write_posts_to_stream(
threads=posts,
stream=stringInputStream,
content_flaggers=content_flagger_names_to_ContentFlaggers(flaggers))
# TODO: use their content flagger selections!
return stringInputStream.getvalue()
def generate_small_example_csv():
results = {}
# Add a specific thread, http://archive.4plebs.org/x/thread/23732801/
results.update(**httpGET_json(gen_thread_api_url('x', 23732801)))
for i in range(1, 10):
# Get the posts from page 1-10 /pol/
results.update(**httpGET_json(gen_index_api_url('pol', i)))
# Add on the posts from page 1 /x/
results.update(**httpGET_json(gen_index_api_url('x', 1)))
# Turn that json dict into a list of Post objects
postList = FourPlebsAPI_Post.from_post_json(results)
# # For all posts from the two index pages (/x/, /pol/)
# for post in postList:
# print(post)
CSVPostWriter.write_posts_to_csv(postList,
'out/post-output-small-example.csv',
ALL_CONTENT_FLAGGERS)
def generate_large_example_csv(page_start=1, page_end=20, boards=['pol', 'x']):
postList = gather_range_with_boards(page_start, page_end, boards)
CSVPostWriter.write_posts_to_csv(postList, 'out/post-output-large.csv',
ALL_CONTENT_FLAGGERS)