def download_asset(temp_dir: str, url: str, filename: str): """Downloads a file in 512 byte chunks from a GitHub octet-stream. :param temp_dir: Temporary directory to be used :type temp_dir: str :param url: URL to the octet-stream API call :type url: str :param filename: Name of the downloaded file :type filename: str """ logging.info("Sending asset download request...") response = requests.send_request(url, "application/octet-stream") response_size = int(response.headers['content-length']) logging.debug(f"Started downloading {response_size} bytes...") with open(os.path.join(temp_dir, filename), "wb") as file: bytes_read = 0 chunk_size = 512 for chunk in response.iter_content(chunk_size): file.write(chunk) bytes_read += chunk_size util.print_progress(bytes_read, response_size, suffix="Downloaded") logging.info(f"Successfully finished downloading {filename}") return True
def _send_to_server(self): data = self.request.data if self.request.content_type == 'application/json' else self.request.form response = send_request( self.request.method, f"https://{config.get_value('server_ip', '')}:5000/{self.path}?{self.request.query_string.decode('utf8')}", data=data, verify=False) return Response(response.content, status=response.status_code, content_type=response.headers['content-type'])
def _response(self): data = self.request.data if self.request.content_type == 'application/json' else self.request.form self.site_name = config.get_value( 'site_address', 'https://redtiger.labs.overthewire.org/') response = send_request( self.request.method, f'{self.site_name}{self.path}?{self.request.query_string.decode("utf8")}', data=data) return Response(response.content, status=response.status_code, content_type=response.headers['content-type'])
def get_latest_release(): """ Returns the JSON object of the latest release of OpenRCT2 :return: JSON object from requests call """ url = f"https://api.github.com/repos/OpenRCT2/OpenRCT2/releases/latest" accept = "application/vnd.github.v3+json" logging.info("Getting latest release from GitHub...") responses = requests.send_request(url, accept) logging.info("Finished getting release info") return responses.json()
def simple_post(domain, url, data, *, headers, auth, verify, notify_addresses=None, payload_id=None): """ POST with a cleaner API, and return the actual HTTPResponse object, so that error codes can be interpreted. """ default_headers = CaseInsensitiveDict({ "content-type": "text/xml", "content-length": str(len(data)), }) default_headers.update(headers) requests = Requests( domain, base_url='', username=None, password=None, verify=verify, notify_addresses=notify_addresses, payload_id=payload_id, ) # Use ``Requests.send_request()`` instead of ``Requests.post()`` to # pass full URL. return requests.send_request('POST', url, data=data, auth=auth, headers=default_headers)
def mirror_cache(*a, **ka): ''' # 'Content-Length': '', pythonAnywhere 这一句 请求头, 会导致绝大数网站返回 400 错误 ''' # us=request.path.split('/') ip = request.headers.get('X-Real-Ip', request.remote_addr) if gblock_ip and ip not in ips: ipsn.append(ip) F.dill_dump(obj=ipsn, file='ipsn') return ip + '\nNot allowed!\n' + U.stime() if gwrite_req: F.write(U.gst + 'q/{}-{}.txt'.format(ip, U.stime()), U.dir(request), mkdir=True) path = request.path[1:] method = request.method # fn=cache_path+method[:1]+T.url2fn(path+request.environ.get('HTTP_COOKIE','')[:99] ) url = T.sub(request.url, request.url_root) # if fn = cache_path + method[:1] + T.url2fn(url)[:255 - 1 - 5] #Linux OSError: [Errno 36] File name too long path 不算长度内。filename+ext 长度<= 255 OK if 'qgb_static' in path: #z_202111 # py.pdb()() return N.flask_html_response( response=make_response(), file=cache_path + path, remove_tag=(), ) if use_cache: target = F.dill_load(fn) if target: # print() # U.p('##',fn,file=sys.stderr,flush=True) return target_to_flask_response(target, flask_request=request) print(U.stime(), path) send_headers = {} for k, v in request.headers.items(): v = v.replace(request.host, target_host) if k in {'Content-Length', 'Content-Type'}: if not v: continue U.println('###', k, v, U.stime(), request.url, request.headers, file=sys.stderr) if k == 'Host' and U.one_in(['uploads', '.jpg', '.png', 'skin/'], path): continue # z_202111 send_headers[k] = v target = send_request( method=method, url=target_base_url + path, params=request.args, headers=send_headers, ) flask_response = target_to_flask_response(target, flask_request=request) if dump: F.dill_dump(protocol=4, obj=target, file=fn) return flask_response
def request( self, uri, method=HttpMethod.GET, raise_on_exc=False, dump_body=None, chunk_size=None, decode_unicode=False, **kwargs, ): """ :param uri: :param method: :param raise_on_exc: :param dump_body: :param chunk_size: :param decode_unicode: :param kwargs: :return: """ kwargs["auth"] = self.get_auth() if dump_body is None: dump_body = self._dump_body else: dump_body = self._normalize_dump_flag(dump_body) if kwargs.get("stream") is True: # if stream not dump response body dump_body = (dump_body[0], False) try: kwargs.setdefault("timeout", self._timeout) url = self.normalize_url(uri) req = ObjectDict(method=method, url=url, **kwargs) self._logger.info("%s", self.dump_request(req, dump_body[0])) response = send_request(method, self.normalize_url(uri), **kwargs) except NetworkError as exc: self._logger.exception(exc) if raise_on_exc or self._raise_on_exc: raise # pragma: no cover return self.prepare_response(status=httpcode.SERVICE_UNAVAILABLE, exception=exc) log_resp = self.dump_response(response, dump_body[1]) try: response.raise_for_status() self._logger.info("%s", log_resp) except HTTPStatusError as exc: self._logger.warning("%s", log_resp) response = exc.response if raise_on_exc or self._raise_on_exc: raise if kwargs.get("stream") is True: body = response.iter_content(chunk_size, decode_unicode) elif "json" in (response.headers.get("Content-Type") or ""): body = response.json() else: body = response.text return self.prepare_response(body=body, status=response.status_code, headers=dict(response.headers))