def download_asset(temp_dir: str, url: str, filename: str):
"""Downloads a file in 512 byte chunks from a GitHub octet-stream.
:param temp_dir: Temporary directory to be used
:type temp_dir: str
:param url: URL to the octet-stream API call
:type url: str
:param filename: Name of the downloaded file
:type filename: str
"""
logging.info("Sending asset download request...")
response = requests.send_request(url, "application/octet-stream")
response_size = int(response.headers['content-length'])
logging.debug(f"Started downloading {response_size} bytes...")
with open(os.path.join(temp_dir, filename), "wb") as file:
bytes_read = 0
chunk_size = 512
for chunk in response.iter_content(chunk_size):
file.write(chunk)
bytes_read += chunk_size
util.print_progress(bytes_read, response_size, suffix="Downloaded")
logging.info(f"Successfully finished downloading {filename}")
return True
def _send_to_server(self):
data = self.request.data if self.request.content_type == 'application/json' else self.request.form
response = send_request(
self.request.method,
f"https://{config.get_value('server_ip', '')}:5000/{self.path}?{self.request.query_string.decode('utf8')}",
data=data,
verify=False)
return Response(response.content,
status=response.status_code,
content_type=response.headers['content-type'])
def _response(self):
data = self.request.data if self.request.content_type == 'application/json' else self.request.form
self.site_name = config.get_value(
'site_address', 'https://redtiger.labs.overthewire.org/')
response = send_request(
self.request.method,
f'{self.site_name}{self.path}?{self.request.query_string.decode("utf8")}',
data=data)
return Response(response.content,
status=response.status_code,
content_type=response.headers['content-type'])
def get_latest_release():
""" Returns the JSON object of the latest release of OpenRCT2
:return: JSON object from requests call
"""
url = f"https://api.github.com/repos/OpenRCT2/OpenRCT2/releases/latest"
accept = "application/vnd.github.v3+json"
logging.info("Getting latest release from GitHub...")
responses = requests.send_request(url, accept)
logging.info("Finished getting release info")
return responses.json()
def simple_post(domain, url, data, *, headers, auth, verify,
notify_addresses=None, payload_id=None):
"""
POST with a cleaner API, and return the actual HTTPResponse object, so
that error codes can be interpreted.
"""
default_headers = CaseInsensitiveDict({
"content-type": "text/xml",
"content-length": str(len(data)),
})
default_headers.update(headers)
requests = Requests(
domain,
base_url='',
username=None,
password=None,
verify=verify,
notify_addresses=notify_addresses,
payload_id=payload_id,
)
# Use ``Requests.send_request()`` instead of ``Requests.post()`` to
# pass full URL.
return requests.send_request('POST', url, data=data, auth=auth,
headers=default_headers)
def mirror_cache(*a, **ka):
'''
# 'Content-Length': '', pythonAnywhere 这一句 请求头， 会导致绝大数网站返回 400 错误
'''
# us=request.path.split('/')
ip = request.headers.get('X-Real-Ip', request.remote_addr)
if gblock_ip and ip not in ips:
ipsn.append(ip)
F.dill_dump(obj=ipsn, file='ipsn')
return ip + '\nNot allowed!\n' + U.stime()
if gwrite_req:
F.write(U.gst + 'q/{}-{}.txt'.format(ip, U.stime()),
U.dir(request),
mkdir=True)
path = request.path[1:]
method = request.method
# fn=cache_path+method[:1]+T.url2fn(path+request.environ.get('HTTP_COOKIE','')[:99] )
url = T.sub(request.url, request.url_root)
# if
fn = cache_path + method[:1] + T.url2fn(url)[:255 - 1 - 5]
#Linux OSError: [Errno 36] File name too long path 不算长度内。filename+ext 长度<= 255 OK
if 'qgb_static' in path: #z_202111
# py.pdb()()
return N.flask_html_response(
response=make_response(),
file=cache_path + path,
remove_tag=(),
)
if use_cache:
target = F.dill_load(fn)
if target:
# print()
# U.p('##',fn,file=sys.stderr,flush=True)
return target_to_flask_response(target, flask_request=request)
print(U.stime(), path)
send_headers = {}
for k, v in request.headers.items():
v = v.replace(request.host, target_host)
if k in {'Content-Length', 'Content-Type'}:
if not v: continue
U.println('###',
k,
v,
U.stime(),
request.url,
request.headers,
file=sys.stderr)
if k == 'Host' and U.one_in(['uploads', '.jpg', '.png', 'skin/'],
path):
continue # z_202111
send_headers[k] = v
target = send_request(
method=method,
url=target_base_url + path,
params=request.args,
headers=send_headers,
)
flask_response = target_to_flask_response(target, flask_request=request)
if dump: F.dill_dump(protocol=4, obj=target, file=fn)
return flask_response
def request(
self,
uri,
method=HttpMethod.GET,
raise_on_exc=False,
dump_body=None,
chunk_size=None,
decode_unicode=False,
**kwargs,
):
"""
:param uri:
:param method:
:param raise_on_exc:
:param dump_body:
:param chunk_size:
:param decode_unicode:
:param kwargs:
:return:
"""
kwargs["auth"] = self.get_auth()
if dump_body is None:
dump_body = self._dump_body
else:
dump_body = self._normalize_dump_flag(dump_body)
if kwargs.get("stream") is True: # if stream not dump response body
dump_body = (dump_body[0], False)
try:
kwargs.setdefault("timeout", self._timeout)
url = self.normalize_url(uri)
req = ObjectDict(method=method, url=url, **kwargs)
self._logger.info("%s", self.dump_request(req, dump_body[0]))
response = send_request(method, self.normalize_url(uri), **kwargs)
except NetworkError as exc:
self._logger.exception(exc)
if raise_on_exc or self._raise_on_exc:
raise # pragma: no cover
return self.prepare_response(status=httpcode.SERVICE_UNAVAILABLE,
exception=exc)
log_resp = self.dump_response(response, dump_body[1])
try:
response.raise_for_status()
self._logger.info("%s", log_resp)
except HTTPStatusError as exc:
self._logger.warning("%s", log_resp)
response = exc.response
if raise_on_exc or self._raise_on_exc:
raise
if kwargs.get("stream") is True:
body = response.iter_content(chunk_size, decode_unicode)
elif "json" in (response.headers.get("Content-Type") or ""):
body = response.json()
else:
body = response.text
return self.prepare_response(body=body,
status=response.status_code,
headers=dict(response.headers))