def program(
args: argparse.Namespace,
env: Environment,
) -> ExitStatus:
"""
The main program without error handling.
"""
exit_status = ExitStatus.SUCCESS
downloader = None
try:
if args.download:
args.follow = True # --download implies --follow.
downloader = Downloader(output_file=args.output_file,
progress_file=env.stderr,
resume=args.download_resume)
downloader.pre_request(args.headers)
initial_request = None
final_response = None
for message in collect_messages(args, env.config.directory):
write_message(
requests_message=message,
env=env,
args=args,
)
if isinstance(message, requests.PreparedRequest):
if not initial_request:
initial_request = message
else:
final_response = message
if args.check_status or downloader:
exit_status = http_status_to_exit_status(
http_status=message.status_code, follow=args.follow)
if (not env.stdout_isatty
and exit_status != ExitStatus.SUCCESS):
env.log_error(
f'HTTP {message.raw.status} {message.raw.reason}',
level='warning')
if downloader and exit_status == ExitStatus.SUCCESS:
# Last response body download.
download_stream, download_to = downloader.start(
initial_url=initial_request.url,
final_response=final_response,
)
write_stream(
stream=download_stream,
outfile=download_to,
flush=False,
)
downloader.finish()
if downloader.interrupted:
exit_status = ExitStatus.ERROR
env.log_error('Incomplete download: size=%d; downloaded=%d' %
(downloader.status.total_size,
downloader.status.downloaded))
return exit_status
finally:
if downloader and not downloader.finished:
downloader.failed()
if (not isinstance(args, list) and args.output_file
and args.output_file_specified):
args.output_file.close()
def program(
args: argparse.Namespace,
env: Environment,
) -> ExitStatus:
"""
The main program without error handling.
"""
exit_status = ExitStatus.SUCCESS
downloader = None
try:
if args.download:
args.follow = True # --download implies --follow.
downloader = Downloader(output_file=args.output_file,
progress_file=env.stderr,
resume=args.download_resume)
downloader.pre_request(args.headers)
needs_separator = False
def maybe_separate():
nonlocal needs_separator
if env.stdout_isatty and needs_separator:
needs_separator = False
getattr(env.stdout, 'buffer', env.stdout).write(b'\n\n')
initial_request: Optional[requests.PreparedRequest] = None
final_response: Optional[requests.Response] = None
def request_body_read_callback(chunk: bytes):
should_pipe_to_stdout = (
# Request body output desired
OUT_REQ_BODY in args.output_options
# & not `.read()` already pre-request (e.g., for compression)
and initial_request
# & non-EOF chunk
and chunk)
if should_pipe_to_stdout:
msg = requests.PreparedRequest()
msg.is_body_upload_chunk = True
msg.body = chunk
msg.headers = initial_request.headers
write_message(requests_message=msg,
env=env,
args=args,
with_body=True,
with_headers=False)
messages = collect_messages(
args=args,
config_dir=env.config.directory,
request_body_read_callback=request_body_read_callback)
for message in messages:
maybe_separate()
is_request = isinstance(message, requests.PreparedRequest)
with_headers, with_body = get_output_options(args=args,
message=message)
if is_request:
if not initial_request:
initial_request = message
is_streamed_upload = not isinstance(
message.body, (str, bytes))
if with_body:
with_body = not is_streamed_upload
needs_separator = is_streamed_upload
else:
final_response = message
if args.check_status or downloader:
exit_status = http_status_to_exit_status(
http_status=message.status_code, follow=args.follow)
if (not env.stdout_isatty
and exit_status != ExitStatus.SUCCESS):
env.log_error(
f'HTTP {message.raw.status} {message.raw.reason}',
level='warning')
write_message(
requests_message=message,
env=env,
args=args,
with_headers=with_headers,
with_body=with_body,
)
maybe_separate()
if downloader and exit_status == ExitStatus.SUCCESS:
# Last response body download.
download_stream, download_to = downloader.start(
initial_url=initial_request.url,
final_response=final_response,
)
write_stream(
stream=download_stream,
outfile=download_to,
flush=False,
)
downloader.finish()
if downloader.interrupted:
exit_status = ExitStatus.ERROR
env.log_error('Incomplete download: size=%d; downloaded=%d' %
(downloader.status.total_size,
downloader.status.downloaded))
return exit_status
finally:
if downloader and not downloader.finished:
downloader.failed()
if (not isinstance(args, list) and args.output_file
and args.output_file_specified):
args.output_file.close()
def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
"""
The main program without error handling.
"""
# TODO: Refactor and drastically simplify, especially so that the separator logic is elsewhere.
exit_status = ExitStatus.SUCCESS
downloader = None
initial_request: Optional[requests.PreparedRequest] = None
final_response: Optional[requests.Response] = None
def separate():
getattr(env.stdout, 'buffer',
env.stdout).write(MESSAGE_SEPARATOR_BYTES)
def request_body_read_callback(chunk: bytes):
should_pipe_to_stdout = bool(
# Request body output desired
OUT_REQ_BODY in args.output_options
# & not `.read()` already pre-request (e.g., for compression)
and initial_request
# & non-EOF chunk
and chunk)
if should_pipe_to_stdout:
msg = requests.PreparedRequest()
msg.is_body_upload_chunk = True
msg.body = chunk
msg.headers = initial_request.headers
write_message(requests_message=msg,
env=env,
args=args,
with_body=True,
with_headers=False)
try:
if args.download:
args.follow = True # --download implies --follow.
downloader = Downloader(output_file=args.output_file,
progress_file=env.stderr,
resume=args.download_resume)
downloader.pre_request(args.headers)
messages = collect_messages(
args=args,
config_dir=env.config.directory,
request_body_read_callback=request_body_read_callback)
force_separator = False
prev_with_body = False
if args.output_format_form == "RAW":
# Process messages as they’re generated
for message in messages:
is_request = isinstance(message, requests.PreparedRequest)
with_headers, with_body = get_output_options(args=args,
message=message)
do_write_body = with_body
if prev_with_body and (with_headers or with_body) and (
force_separator or not env.stdout_isatty):
# Separate after a previous message with body, if needed. See test_tokens.py.
separate()
force_separator = False
if is_request:
if not initial_request:
initial_request = message
is_streamed_upload = not isinstance(
message.body, (str, bytes))
if with_body:
do_write_body = not is_streamed_upload
force_separator = is_streamed_upload and env.stdout_isatty
else:
final_response = message
if args.check_status or downloader:
exit_status = http_status_to_exit_status(
http_status=message.status_code,
follow=args.follow)
if exit_status != ExitStatus.SUCCESS and (
not env.stdout_isatty or args.quiet):
env.log_error(
f'HTTP {message.raw.status} {message.raw.reason}',
level='warning')
write_message(requests_message=message,
env=env,
args=args,
with_headers=with_headers,
with_body=do_write_body)
prev_with_body = with_body
else:
all_messages_together = []
for message in messages:
is_request = isinstance(message, requests.PreparedRequest)
with_headers, with_body = get_output_options(args=args,
message=message)
#force_separator = False
if is_request:
with_headers_req = with_headers
with_body_req = with_body
if not initial_request:
initial_request = message
is_streamed_upload = not isinstance(
message.body, (str, bytes))
if with_body:
with_body_req = not is_streamed_upload
#force_separator = is_streamed_upload and env.stdout_isatty
else:
with_headers_res = with_headers
with_body_res = with_body
final_response = message
if args.check_status or downloader:
exit_status = http_status_to_exit_status(
http_status=message.status_code,
follow=args.follow)
if exit_status != ExitStatus.SUCCESS and (
not env.stdout_isatty or args.quiet):
env.log_error(
f'HTTP {message.raw.status} {message.raw.reason}',
level='warning')
all_messages_together.append(message)
write_message_json(requests_message=all_messages_together,
env=env,
args=args,
with_headers_req=with_headers_req,
with_body_req=with_body_req,
with_headers_res=with_headers_res,
with_body_res=with_body_res)
prev_with_body = with_body
# Cleanup
if force_separator:
separate()
if downloader and exit_status == ExitStatus.SUCCESS:
# Last response body download.
download_stream, download_to = downloader.start(
initial_url=initial_request.url,
final_response=final_response,
)
write_stream(stream=download_stream,
outfile=download_to,
flush=False)
downloader.finish()
if downloader.interrupted:
exit_status = ExitStatus.ERROR
env.log_error('Incomplete download: size=%d; downloaded=%d' %
(downloader.status.total_size,
downloader.status.downloaded))
return exit_status
finally:
if downloader and not downloader.finished:
downloader.failed()
if not isinstance(
args,
list) and args.output_file and args.output_file_specified:
args.output_file.close()