def encodecaps(caps): return b'\n'.join( b'%s=%s' % (quote_from_bytes(k).encode('ascii'), b','.join( quote_from_bytes(v).encode('ascii') for v in values)) if values else quote_from_bytes(k).encode('ascii' ) for k, values in sorted(caps.items()))
def rewrite(self, path, method = None, keepresponse = True): "Rewrite this request to another processor. Must be called before header sent" if self._sendHeaders: raise HttpProtocolException('Cannot modify response, headers already sent') if getattr(self.event, 'rewritedepth', 0) >= getattr(self.protocol, 'rewritedepthlimit', 32): raise HttpRewriteLoopException newpath = urljoin(quote_from_bytes(self.path).encode('ascii'), path) if newpath == self.fullpath or newpath == self.originalpath: raise HttpRewriteLoopException extraparams = {} if keepresponse: if hasattr(self, 'status'): extraparams['status'] = self.status extraparams['sent_headers'] = self.sent_headers extraparams['sent_cookies'] = self.sent_cookies r = HttpRequestEvent(self.host, newpath, self.method if method is None else method, self.connection, self.connmark, self.xid, self.protocol, headers = self.headers, headerdict = self.headerdict, setcookies = self.setcookies, stream = self.inputstream, rewritefrom = self.fullpath, originalpath = self.originalpath, rewritedepth = getattr(self.event, 'rewritedepth', 0) + 1, **extraparams ) for m in self.connection.waitForSend(r): yield m self._sendHeaders = True self.outputstream = None
def trackerhash(type): """Generate a random info_hash to be used with this tracker.""" t_hash = urandom(20) if type == 'udp': return t_hash if type == 'http': return quote_from_bytes(t_hash)
def to_datauri(mimetype, data, charset: str = 'utf-8', base64: bool = False, binary: bool = True): """ Convert data to data URI. :param mimetype: MIME types (e.g. 'text/plain','image/png' etc.) :param data: Data representations. :param charset: Charset may be any character set registered with IANA :param base64: Used to encode arbitrary octet sequences into a form that satisfies the rules of 7bit. Designed to be efficient for non-text 8 bit and binary data. Sometimes used for text data that frequently uses non-US-ASCII characters. :param binary: :return: """ parts = ['data:', mimetype] if charset is not None: parts.extend([';charset=', charset]) if base64: parts.append(';base64') from base64 import encodebytes as encode64 if binary: encoded_data = encode64(data).decode(charset).replace('\n', '').strip() else: encoded_data = encode64(data).strip() else: from urllib.parse import quote_from_bytes, quote if binary: encoded_data = quote_from_bytes(data) else: encoded_data = quote(data) parts.extend([',', encoded_data]) return ''.join(parts)
def capable(self, capability): if capability == b'bundle2': return quote_from_bytes(HgRepoHelper.capable(b'bundle2') or b'').encode('ascii') if capability in (b'clonebundles', b'cinnabarclone'): return HgRepoHelper.capable(capability) is not None return capability in (b'getbundle', b'unbundle', b'lookup')
def __repr__(self): try: return self._representation except AttributeError: pass try: representation = self.string except AttributeError: representation = None else: import re if not re.fullmatch(r'[\x20!-~]+', representation): representation = None else: prefix = self._CODECS[self.codec] if prefix != 's' or self.requires_prefix(): representation = F'{prefix}:{representation}' if representation is None: printable = (B'0123456789' B'!#$%&()*,-./:;=?@[\\]^{}~' B'abcdefghijklmnopqrstuvwxyz' B'ABCDEFGHIJKLMNOPQRSTUVWXYZ') p = sum(1 for c in self if c in printable) if p >= len(self) * 0.8: from urllib.parse import quote_from_bytes quoted = quote_from_bytes(self, printable) representation = F'q:{quoted}' else: representation = F'h:{self.hex()}' self._representation = representation return representation
def send_encoded(self, url, data): url2 = urlparse(url) host = url2.netloc path = url2.path or '/' query = '?' + url2.query if url2.query else '' request = path + query data2 = '&'.join( (quote(k) + '=' + quote_from_bytes(str(v).encode('utf8')) for k, v in data.items())).encode('ascii') try: http = HTTPConnection(host) http.putrequest('POST', request) http.putheader('Content-Type', 'application/x-www-form-urlencoded') http.putheader('User-Agent', 'Fluxid MOC Scrobbler 0.2') http.putheader('Content-Length', str(len(data2))) http.endheaders() http.send(data2) response = http.getresponse().read().decode('utf8').upper().strip() except Exception as e: raise HardErrorException(str(e)) if response == 'BADSESSION': raise BadSessionException elif response.startswith('FAILED'): raise FailedException( response.split(' ', 1)[1].strip() + (' POST = [%r]' % data2))
def send_encoded(self, url, data): url2 = urlparse(url) host = url2.netloc path = url2.path or '/' query = '?' + url2.query if url2.query else '' request = path + query data2 = '&'.join(( quote(k) + '=' + quote_from_bytes(str(v).encode('utf8')) for k, v in data.items() )).encode('ascii') try: http = HTTPConnection(host) http.putrequest('POST', request) http.putheader('Content-Type', 'application/x-www-form-urlencoded') http.putheader('User-Agent', 'Fluxid MOC Scrobbler 0.2') http.putheader('Content-Length', str(len(data2))) http.endheaders() http.send(data2) response = http.getresponse().read().decode('utf8').upper().strip() except Exception as e: raise HardErrorException(str(e)) if response == 'BADSESSION': raise BadSessionException elif response.startswith('FAILED'): raise FailedException(response.split(' ', 1)[1].strip() + (' POST = [%r]' % data2))
async def save_message(self, buf: bytes) -> None: async with aiofiles.open(self.message_filename, 'at') as f: if self.message_style == 'urlencode': await f.write(quote_from_bytes(buf) + '\n') else: await f.write(buf) f.flush()
def escapeOnce(input): """escape everything outside of 32-128, except #""" if input: return quote_from_bytes( input, safe=br'''!"$&'()*+,-./:;<=>?@[\]^_`{|}~''').encode('ascii') else: return input
def main() -> None: if not sys.stdout.isatty() and '--pretty' not in sys.argv: os.execlp('rg', 'rg', *sys.argv[1:]) cmdline = ['rg', '--pretty'] + sys.argv[1:] p = subprocess.Popen(cmdline, stdout=subprocess.PIPE) assert p.stdout is not None write: Callable[[bytes], None] = cast(Callable[[bytes], None], sys.stdout.buffer.write) sgr_pat = re.compile(br'\x1b\[.*?m') osc_pat = re.compile(b'\x1b\\].*?\x1b\\\\') num_pat = re.compile(b'^(\\d+):') in_result: bytes = b'' hostname = socket.gethostname().encode('utf-8') for line in p.stdout: line = osc_pat.sub(b'', line) # remove any existing hyperlinks clean_line = sgr_pat.sub(b'', line).rstrip() # remove SGR formatting if not clean_line: in_result = b'' write(b'\n') continue if in_result: m = num_pat.match(clean_line) if m is not None: write_hyperlink(write, in_result, line, frag=m.group(1)) else: if line.strip(): path = quote_from_bytes(os.path.abspath(clean_line)).encode('utf-8') in_result = b'file://' + hostname + path write_hyperlink(write, in_result, line) else: write(line)
def main(args) -> None: if not sys.stdout.isatty() and '--pretty' not in args: os.execlp('fd', 'fd', *args[1:]) cmdline = ['fd', '--color', 'always'] + args[1:] p = subprocess.Popen(cmdline, stdout=subprocess.PIPE) assert p.stdout is not None write: Callable[[bytes], None] = cast(Callable[[bytes], None], sys.stdout.buffer.write) sgr_pat = re.compile(br'\x1b\[.*?m') osc_pat = re.compile(b'\x1b\\].*?\x1b\\\\') num_pat = re.compile(b'^(\\d+):') hostname = socket.gethostname().encode('utf-8') try: for line in p.stdout: line = osc_pat.sub(b'', line) # remove any existing hyperlinks clean_line = sgr_pat.sub(b'', line).rstrip() # remove SGR formatting if not clean_line: in_result = b'' write(b'\n') continue if line.strip(): path = quote_from_bytes(os.path.abspath(clean_line)).encode('utf-8') in_result = b'file://' + hostname + path write_hyperlink(write, in_result, line) else: write(line) except KeyboardInterrupt: p.send_signal(signal.SIGINT) p.stdout.close() except EOFError: p.stdout.close() raise SystemExit(p.wait())
def escapeOnce(input): """escape everything outside of 32-128, except #""" if input: return quote_from_bytes( input, safe=b'''!"$&'()*+,-./:;<=>?@[\]^_`{|}~''').encode( 'ascii') else: return input
def test_type_convert(self): expected = { 'p0': 'v0', 'p1': 1, 'p2': 0, 'p3': 2, 'p4': 3.141, 'p5': 2.718, 'p6': b'\x00Hello\x00', 'p8': True, 'p9': datetime.date(2017, 12, 1), 'p10': datetime.datetime(2017, 12, 1, 6, 12, 34, 56789, datetime.timezone.utc), 'p11': { 'num': 123, 'str': 'hoge', 'foo': 3.14 }, 'p12': [False, True], 'p13': [1, 2, 3], 'p14': { 'foo': 321, 'bar': False, } } q = {k: str(v) for k, v in expected.items()} q['p6'] = base64.b64encode(expected['p6']) q['p9'] = expected['p9'].isoformat() q['p10'] = expected['p10'].isoformat() del q['p11'] del q['p13'] q.update({ 'p11[{}]'.format(quote_plus(k)): quote_plus(str(v)) for k, v in expected['p11'].items() }) q['p12'] = ','.join([str(v).lower() for v in expected['p12']]) q['p14'] = ','.join( ['{},{}'.format(k, v) for k, v in expected['p14'].items()]) qs = [] for k, v in q.items(): qs.append('{}={}'.format( quote_plus(k), quote_plus(v) if isinstance(v, str) else quote_from_bytes(v))) for v in expected['p13']: qs.append('p13={}'.format(v)) res = self._get('/test_type_convert?' + '&'.join(qs), status=200) self.assertEqual(res, expected) self.assertEqual(self._get('/test_type_convert', status=200), {})
def make_uri(self, path): bpath = bytes(path) if bpath.startswith(b':/'): bpath = bpath[2:] elif path.startswith(b':'): bpath = bpath[1:] return 'qrc:/' + quote_from_bytes(bpath)
def EncryptMessage(key, msg): quoted_msg = quote_from_bytes(msg).encode() full_msg = (b"comment1=cooking%20MCs;userdata=" + quoted_msg + b";comment2=%20like%20a%20pound%20of%20bacon") print([full_msg[i:i + 16] for i in range(0, len(full_msg), 16)]) iv = GenerateRandomIV() ciphertext = CBCEncrypt(16, full_msg, AES.new(key, AES.MODE_ECB), iv) return {"iv": iv, "ciphertext": ciphertext}
def redirect(self, path, status=302): location = urljoin( urlunsplit((b'https' if self.https else b'http', self.host, quote_from_bytes(self.path).encode('ascii'), '', '')), path) self.startResponse(status, [(b'Location', location)]) for m in self.write(b'<a href="' + self.escape(location, True) + b'">' + self.escape(location) + b'</a>'): yield m for m in self.flush(True): yield m
def url_encode(cls, url): """ URL转码 :param url: :return: """ if isinstance(url, str): res = parse.quote(url) else: res = parse.quote_from_bytes(url) return res
async def redirect(self, path, status=302): """ Redirect this request with 3xx status """ location = urljoin( urlunsplit((b'https' if self.https else b'http', self.host, quote_from_bytes(self.path).encode('ascii'), '', '')), path) self.start_response(status, [(b'Location', location)]) await self.write(b'<a href="' + self.escape(location, True) + b'">' + self.escape(location) + b'</a>') await self.flush(True)
def main(): r = requests.get(AUTH_URL, {"cmd": "ls"}, allow_redirects=False) orig_signature = r.cookies["signature"] h = octothorpe(_state=bytearray.fromhex(orig_signature), _length=128) h.update(b"&cmd=" + CMD) forged_signature = h.hexdigest() expiry = int( (datetime.now(timezone.utc) + timedelta(seconds=15)).timestamp()) expiry_arg = b'expiry=' + str(expiry).encode() + b'&' # We got it statically, but it is calculatble (see octothorpe::_finalize) concated_bytes = b"\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc0\x01\x00\x00\x00\x00\x00\x00" forged_url = RUN_URL + b"?" + expiry_arg + b"cmd=ls" + quote_from_bytes( concated_bytes).encode() + b"&cmd=" + quote_from_bytes(CMD).encode() r = requests.get(forged_url, cookies={"signature": forged_signature}) print(r, r.content) if r.status_code == 200: res = json.loads(r.content) print(base64.b64decode(res["stdout"])) print(base64.b64decode(res["stderr"]))
def test_non_ascii_in_GET_params(self): @wptserve.handlers.handler def handler(request, response): return request.GET[b"foo"] route = ("GET", "/test/test_unicode_in_get", handler) self.server.router.register(*route) # We intentionally choose an encoding that's not the default UTF-8. encoded_text = u"どうも".encode("shift-jis") quoted = quote_from_bytes(encoded_text) resp = self.request(route[1], query="foo=" + quoted) self.assertEqual(encoded_text, resp.read())
def main() -> None: if not sys.stdout.isatty( ) and '--pretty' not in sys.argv and '-p' not in sys.argv: os.execlp('rg', 'rg', *sys.argv[1:]) cmdline = ['rg', '--pretty', '--with-filename'] + sys.argv[1:] try: p = subprocess.Popen(cmdline, stdout=subprocess.PIPE) except FileNotFoundError: raise SystemExit( 'Could not find the rg executable in your PATH. Is ripgrep installed?' ) assert p.stdout is not None write: Callable[[bytes], None] = cast(Callable[[bytes], None], sys.stdout.buffer.write) sgr_pat = re.compile(br'\x1b\[.*?m') osc_pat = re.compile(b'\x1b\\].*?\x1b\\\\') num_pat = re.compile(br'^(\d+)[:-]') in_result: bytes = b'' hostname = socket.gethostname().encode('utf-8') try: for line in p.stdout: line = osc_pat.sub(b'', line) # remove any existing hyperlinks clean_line = sgr_pat.sub(b'', line).rstrip() # remove SGR formatting if not clean_line: in_result = b'' write(b'\n') continue if in_result: m = num_pat.match(clean_line) if m is not None: write_hyperlink(write, in_result, line, frag=m.group(1)) else: write(line) else: if line.strip(): path = quote_from_bytes( os.path.abspath(clean_line)).encode('utf-8') in_result = b'file://' + hostname + path write_hyperlink(write, in_result, line) else: write(line) except KeyboardInterrupt: p.send_signal(signal.SIGINT) except (EOFError, BrokenPipeError): pass finally: p.stdout.close() raise SystemExit(p.wait())
def submit(self, feature, description): """Submit feature to server for evaluation on test data. If successful, registers feature in feature database and returns key performance metrics. Runs the feature in an isolated environment to extract the feature values. Validates the feature values. Then, builds a model on that one feature, performs cross validation, and returns key performance metrics. Parameters ---------- feature : function Feature to evaluate description : str Feature description """ from featurehub.user.session import Session feature_dill = quote_from_bytes(dill.dumps(feature)) code = get_source(feature) data = { "database": self.orm.database, "problem_id": self.problem_id, "feature_dill": feature_dill, "code": code, "description": description, } response = Session._eval_server_post("submit", data) if response.ok: try: eval_response = EvaluationResponse.from_string(response.text) print(eval_response) except Exception as e: # TODO print("response failed with exception") print(traceback.format_exc(), file=sys.stderr) try: print(response, file=sys.stderr) print(response.text, file=sys.stderr) except Exception: pass else: # TODO print("response failed with bad status code") try: print(response, file=sys.stderr) print(response.text, file=sys.stderr) except Exception: pass
def construct_file_uri(location_base_uris: typing.Sequence[str], filename: str) -> str: """ Construct the file URI given ``location_base_uris`` values and a ``filename`` (which should be in POSIX format). Generally, ``filename`` comes from the column of a collection resource. """ # TODO: Support handling multiple "location_base_uris". location_base_uri = location_base_uris[0] # "location_base_uris" should be made so that we can just concat with the filename # ("location_base_uris" end with "/"), but we make sure it is so. location_base_uri = utils.ensure_uri_ends_with_slash(location_base_uri) return location_base_uri + url_parse.quote_from_bytes( bytes(pathlib.PurePosixPath(filename)))
def make_datauri(self, mimetype, buffer): parts = ['data:', mimetype] if self.charset is not None: parts.extend([';charset=', self.charset]) if self.base64: parts.append(';base64') from base64 import encodebytes as encode64 encoded_data = encode64(buffer).decode(self.charset).replace( '\n', '').strip() else: from urllib.parse import quote_from_bytes encoded_data = quote_from_bytes(buffer) parts.extend([',', encoded_data]) return ''.join(parts)
def test_non_ascii_in_POST_params(self): @wptserve.handlers.handler def handler(request, response): return request.POST[b"foo"] route = ("POST", "/test/test_unicode_in_POST", handler) self.server.router.register(*route) # We intentionally choose an encoding that's not the default UTF-8. encoded_text = u"どうも".encode("shift-jis") # After urlencoding, the string should only contain ASCII. quoted = quote_from_bytes(encoded_text).encode("ascii") resp = self.request(route[1], method="POST", body=b"foo=" + quoted) self.assertEqual(encoded_text, resp.read())
def encode_default(obj): if isinstance(obj, JSONBytes): return { '<vlcpjsonencode/urlencoded-bytes>': quote_from_bytes(obj.data) } elif isinstance(obj, bytes): return {'<vlcpjsonencode/urlencoded-bytes>': quote_from_bytes(obj)} elif isinstance(obj, NamedStruct): # Hacked in the internal getstate implementation... state = obj.__getstate__() if state[2] is not obj: return { '<vlcpjsonencode/namedstruct.NamedStruct>': { 'type': state[1], 'data': base64.b64encode(state[0]), 'target': state[2] } } else: return { '<vlcpjsonencode/namedstruct.NamedStruct>': { 'type': state[1], 'data': base64.b64encode(state[0]) } } else: if hasattr(obj, 'jsonencode'): try: key = '<vlcpjsonencode/' + type(obj).__module__ + '.' + type( obj).__name__ + '>' except AttributeError: raise TypeError(repr(obj) + " is not JSON serializable") else: return {key: obj.jsonencode()} else: raise TypeError(repr(obj) + " is not JSON serializable")
async def _submit(self, oj, pid, env, code, captcha): response = await self.open( f"https://{self.netloc}/problem/submit", { "oj": ONLINE_JUDGES[oj], "probNum": pid, "language": env, "share": 0, "source": b64encode(quote_from_bytes(code).encode()), "captcha": captcha, }, {'Content-Type': self.URLENCODE}) data = json.loads(response.read()) if "error" in data: if not data.get("captcha", False): raise AuthError(data["error"]) return data
def uri_from_cli( path_or_uri: str, username: str, cluster_name: str, *, allowed_schemes: Sequence[str] = ("file", "storage"), ) -> URL: if "file" in allowed_schemes and path_or_uri.startswith("~"): path_or_uri = os.path.expanduser(path_or_uri) if path_or_uri.startswith("~"): raise ValueError(f"Cannot expand user for {path_or_uri}") path_or_uri = Path(path_or_uri).as_uri() uri = URL(path_or_uri) else: uri = URL(path_or_uri) # len(uri.scheme) == 1 is a workaround for Windows path like C:/path/to.txt if not uri.scheme or len(uri.scheme) == 1: # Workaround for urllib.parse.urlsplit()'s strange behavior with # URLs like "scheme:123". if re.fullmatch(r"[a-zA-Z0-9+\-.]{2,}:[0-9]+", path_or_uri): uri = URL(f"{path_or_uri}#") elif "file" in allowed_schemes: path = Path(path_or_uri) if path.is_absolute(): uri = URL(path.as_uri()) elif re.fullmatch(r"[0-9]+", path_or_uri): uri = URL(f"file:{path_or_uri}#") else: uri = URL("file:" + quote_from_bytes(path.as_posix().encode())) path_or_uri = str(uri) if not uri.scheme: raise ValueError( f"URI Scheme not specified. " f"Please specify one of {', '.join(allowed_schemes)}.") if uri.scheme not in allowed_schemes: raise ValueError( f"Unsupported URI scheme: {uri.scheme}. " f"Please specify one of {', '.join(allowed_schemes)}.") # Check string representation to detect also trailing "?" and "#". _check_uri_str(path_or_uri, uri.scheme) if uri.scheme == "file": uri = normalize_local_path_uri(uri) elif uri.scheme == "blob": uri = normalize_blob_path_uri(uri, cluster_name) else: uri = _normalize_uri(uri, username, cluster_name) return uri
def scrap(self): url = 'https://jisho.org/search/%23kanji%20' #resp = urllib.request.urlopen(url).read().decode("utf-8") resp = urllib.request.urlopen( url + quote_from_bytes(self.name.encode('utf-8'))).read().decode("utf-8") #writter.hey(resp,'htmltest.html','w',False,True) self.strokes = resp.split('<div class="kanji-details__stroke_count">')[ 1].split('<strong>')[1].split('</strong>')[0] self.frequency = resp.split('<div class="frequency">')[1].split( '<strong>')[1].split('</strong>')[0] self.grade = resp.split('<div class="grade">')[1].split( '<strong>')[1].split('</strong>')[0] self.jlpt = resp.split('<div class="jlpt">')[1].split( '<strong>')[1].split('</strong>')[0] writter.hey(self.toString(), 'kanji_dict.txt', 'a', True, False)
def out(data): data = [d[0] for d in data] b = bytearray(b''.join(data)) pre_out() sys.stdout.flush() if shellcode != '': for x in shellcode: b.append(x) if args.httpencode: b = quote_from_bytes(b) print(b, end='') if not args.httpencode: sys.stdout.buffer.write(b) sys.stdout.flush() post_out() sys.stdout.flush()
async def getClients(session, meta, peer_id): print("Requesting clients from {}".format(meta['announce'])) info = meta['info'] m = hashlib.sha1() m.update(bencode.bencode(info)) print(bencode.bencode(info)) byte = m.digest() hash = parse.quote_from_bytes(byte).strip() payload = {"info_hash": hash, "peer_id": peer_id, "downloaded": 0} time.sleep(3) async with session.get( URL(meta['announce'], encoded=True).with_query(payload)) as res: print(res.url) if res.status != 200: print("Failure") print(await res.text()) exit() return await res.read()
def encode(self, s): return quote_from_bytes(s).encode('ascii')
def path_to_uri(path, scheme=Extension.ext_name): """Convert file path to URI.""" assert isinstance(path, bytes), 'Mopidy paths should be bytes' uripath = quote_from_bytes(os.path.normpath(path)) return urlunsplit((scheme, None, uripath, None, None))