def test_b85_padding(self):
eq = self.assertEqual
eq(base64.b85encode(b"x", pad=True), b'cmMzZ')
eq(base64.b85encode(b"xx", pad=True), b'cz6H+')
eq(base64.b85encode(b"xxx", pad=True), b'czAdK')
eq(base64.b85encode(b"xxxx", pad=True), b'czAet')
eq(base64.b85encode(b"xxxxx", pad=True), b'czAetcmMzZ')
eq(base64.b85decode(b'cmMzZ'), b"x\x00\x00\x00")
eq(base64.b85decode(b'cz6H+'), b"xx\x00\x00")
eq(base64.b85decode(b'czAdK'), b"xxx\x00")
eq(base64.b85decode(b'czAet'), b"xxxx")
eq(base64.b85decode(b'czAetcmMzZ'), b"xxxxx\x00\x00\x00")
def proxy_sdns_query(self):
req_bin, client_sock = self.request
sdns_req = {
"request_id": 1,
"query": base64.b85encode(req_bin).decode("ascii"),
"username": "******",
"auth_token": "testtoken"
}
# TODO: using a new connection for each request; this is expensive...
# TODO: real implementation would require certificate validation
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
with ssl_context.wrap_socket(socket(AF_INET, SOCK_STREAM)) as sdns_sock:
sdns_sock.connect((SDNS_HOST, SDNS_PORT))
# Append trailing newline to indicate the end of the request
payload = bytes(json.dumps(sdns_req) + "\n", "ascii")
print("Sending SDNS request: '{}'".format(payload))
sdns_sock.sendall(payload)
sdns_resp = str(sdns_sock.recv(4096), "ascii")
print("Got SDNS response:")
print(str(sdns_resp))
try:
parsed = json.loads(sdns_resp)
if 'status' not in parsed or parsed['status'] != 0 or 'response' not in parsed:
print("SDNS request failed")
else:
client_sock.sendto(base64.b85decode(parsed['response']), self.client_address)
except json.JSONDecodeError as e:
print("Error: JSON decode failed", e)
def b85encode(data):
"""
Encode binary data to ascii text in base85. Data must be bytes.
"""
if PY2:
raise NotImplementedError("Python 2 can't encode data in base85.")
return base64.b85encode(data).decode('ascii')
def store(self, key, value):
"""
Saves a value persistently to the database.
"""
# NOTE: value must not be <bytes> otherwise BOOM! and moreover our sqlite db always return strings as <str>
entry = b85encode(json.dumps(value, ensure_ascii=False).encode()).decode()
self.plugin.pyload.db.set_storage(self.plugin.classname, key, entry)
def test_b85encode(self):
eq = self.assertEqual
tests = {
b'': b'',
b'www.python.org': b'cXxL#aCvlSZ*DGca%T',
bytes(range(255)): b"""009C61O)~M2nh-c3=Iws5D^j+6crX17#SKH9337X"""
b"""AR!_nBqb&%[email protected]{EG;fCFflSSG&MFiI5|2yJUu=?KtV!7L`6nNNJ&ad"""
b"""OifNtP*GA-R8>}2SXo+ITwPvYU}0ioWMyV&XlZI|Y;A6DaB*^Tbai%j"""
b"""[email protected]>41ejE#<ukdcy;l$Dm3n3<ZJoSmMZprN9p"""
b"""[email protected]|{(sHv)}tgWuEu(7hUw6(UkxVgH!yuH4^z`[email protected]#Kp$P$jQpf%+1cv"""
b"""(9zP<)YaD4*xB0K+}+;a;Njxq<mKk)=;`[email protected]^!4`l`1$(#"""
b"""{Qdp""",
b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"""
b"""[email protected]#0^&*();:<>,. []{}""":
b"""VPa!sWoBn+X=-b1ZEkOHadLBXb#`}nd3r%[email protected][email protected]("""
b"""Q&d$}S6EqEFflSSG&MFiI5{CeBQRbjDkv#CIy^osE+AW7dwl""",
b'no padding..': b'[email protected]!Zf7no',
b'zero compression\x00\x00\x00\x00': b'dS!BNAY*TBaB^jHb7^mG00000',
b'zero compression\x00\x00\x00': b'dS!BNAY*TBaB^jHb7^mG0000',
b"""Boundary:\x00\x00\x00\x00""": b"""LT`0$WMOi7IsgCw00""",
b'Space compr: ': b'Q*dEpWgug3ZE$irARr(h',
b'\xff': b'{{',
b'\xff'*2: b'|Nj',
b'\xff'*3: b'|Ns9',
b'\xff'*4: b'|NsC0',
}
for data, res in tests.items():
eq(base64.b85encode(data), res)
self.check_other_types(base64.b85encode, b"www.python.org",
b'cXxL#aCvlSZ*DGca%T')
def dump_observation(observation, f):
if 'cache' in observation:
observation['cache'] = \
str(base64.b85encode(pickle.dumps(observation['cache'])), 'ascii')
json.dump(observation, f)
f.write("\n")
def sendsafe(self, data):
"""
Encode an arbitrary string into a safe form to pass to print()."""
import base64
data = base64.b85encode(self.as_bytes(data))
return self.sendraw(data)
def serializer(x):
# Serialize
try:
data = dumps(x)
except Exception as ex:
raise SerializationError(ex)
# Transmit with b85 encode: safe characters and no newlines
return (b'+' + base64.b85encode(data)).decode('ascii')
def installer(version=None, installer_path=_path(), template_path=os.path.join(PROJECT_ROOT, "template.py")):
print(
"[generate.installer] Generating installer {} (using {})".format(
os.path.relpath(installer_path, PROJECT_ROOT), "pip" + version if version is not None else "latest"
)
)
# Load our wrapper template
with open(template_path, "r", encoding="utf8") as fp:
WRAPPER_TEMPLATE = fp.read()
# Get all of the versions on PyPI
resp = urllib.request.urlopen("https://pypi.python.org/pypi/pip/json")
data = json.loads(resp.read().decode("utf8"))
versions = sorted(data["releases"].keys(), key=packaging.version.parse)
# Filter our list of versions based on the given specifier
s = packaging.specifiers.SpecifierSet("" if version is None else version)
versions = list(s.filter(versions))
# Select the latest version that matches our specifier is
latest = versions[-1]
# Select the wheel file (we assume there will be only one per release)
file_urls = [(x["url"], x["md5_digest"]) for x in data["releases"][latest] if x["url"].endswith(".whl")]
assert len(file_urls) == 1
url, expected_hash = file_urls[0]
# Fetch the file itself.
data = urllib.request.urlopen(url).read()
assert hashlib.md5(data).hexdigest() == expected_hash
# Write out the wrapper script that will take the place of the zip script
# The reason we need to do this instead of just directly executing the
# zip script is that while Python will happily execute a zip script if
# passed it on the file system, it will not however allow this to work if
# passed it via stdin. This means that this wrapper script is required to
# make ``curl https://...../get-pip.py | python`` continue to work.
print("[generate.installer] Write the wrapper script with the bundled zip " "file")
zipdata = base64.b85encode(data).decode("utf8")
chunked = []
for i in range(0, len(zipdata), 79):
chunked.append(zipdata[i : i + 79])
os.makedirs(os.path.dirname(installer_path), exist_ok=True)
with open(installer_path, "w") as fp:
fp.write(WRAPPER_TEMPLATE.format(version="" if version is None else version, zipfile="\n".join(chunked)))
# Ensure the permissions on the newly created file
oldmode = os.stat(installer_path).st_mode & 0o7777
newmode = (oldmode | 0o555) & 0o7777
os.chmod(installer_path, newmode)
print("[generate.installer] Generated installer")
def genCurveTweet(curveHash): tw = "Entropy for curve "+curveHash+" : " tw += createCE() + " " tw = tw[0:140] randlen = 140 - len(tw) randomS = str(base64.b85encode(os.urandom(randlen))) randomS = randomS[2:len(randomS) - 1] randomS = randomS[0:randlen] tw += randomS return tw
def generate_password(passphrase, resetCount, privateKeyHash, domain,
allowSymbols, length):
resetCountString = str(resetCount) if resetCount > 0 else ''
secret = (passphrase + resetCountString + privateKeyHash).encode('utf-8')
key = pbkdf2_hmac('sha512', secret, domain.encode('utf-8'), 100)
if allowSymbols:
return b85encode(key).decode('ascii')[:length]
else:
b64 = b64encode(key).decode('ascii')
return re.sub(r'[\W_]+', '', b64).ljust(len(b64), '0')[:length]
def tojson(self, obj):
'''Hook for json.dump()'''
onam = obj.__class__.__name__
m = { 'clsname': onam }
if onam in ("bytes", "bytearray"):
import base64
s = base64.b85encode( obj ).decode("ascii")
o = Jsonator.JsonBytesWrapper( s, onam )
return o
m.update(obj.__dict__)
return m
def enc(s):
labels = []
labels.append('s')
# print(s)
# print("l: ", s.__len__())
# print("..")
b = str.encode(s, 'ascii')
labels.append('b')
# print(b)
# print("l: ", b.__len__())
# print("..")
b64 = base64.standard_b64encode(b)
labels.append('b64')
# print(b64)
# print("l: ", b64.__len__())
# print("..")
z = gzip.compress(b)
labels.append('z')
# print(z)
# print("l: ", z.__len__())
# print("..")
z64 = base64.standard_b64encode(z)
labels.append('z64')
# print(z64)
# print("l: ", z64.__len__())
# print("..")
z85 = base64.b85encode(z)
labels.append('z85')
# print(z85)
# print("l: ", z85.__len__())
return (
(
s.__len__(),
b.__len__(),
b64.__len__(),
z.__len__(),
z64.__len__(),
z85.__len__()
),
labels
)
def base85_digest(hash_):
'''
Get base 85 encoded digest of hash
Parameters
----------
hash_ : hash
hashlib hash object. E.g. the return of hashlib.sha512()
Returns
-------
str
base 85 encoded digest
'''
return base64.b85encode(hash_.digest()).decode('ascii')
def handle(self):
request = self.rfile.readline().strip().decode("ascii")
print("Got request from {}: '{}'".format(self.client_address, request))
try:
parsed = json.loads(request)
dns_query = DNSRecord.parse(base64.b85decode(parsed['query']))
except json.JSONDecodeError as e:
print("Error: JSON decode failed", e)
self.send_error_rsp("Invalid JSON")
return
except DNSError as e:
print("Error: DNS record decode failed", e)
self.send_error_rsp("Invalid DNS query")
return
# Only looking at first question part
q = dns_query.get_q()
if q.qtype != QTYPE.A:
print("Error: Unexpected query type {} (only A/IPv4 lookup supported)".format(q.qtype))
self.send_error_rsp("Invalid query type")
return
# Note: this is a very simplistic implementation that only returns A records
hostname = q.qname.idna()
dns_response = dns_query.reply()
if hostname in DNS_RECORDS:
virt_addr = DNS_RECORDS[hostname]
# TODO: would generate virtual IP here and communicate with OF controller to install mapping to private IP;
# for the simulation, we are hard-coding this part and not implementing communication with the OF controller
dns_response.add_answer(RR(rname=hostname, rtype=QTYPE.A, ttl=DNS_TTL, rdata=A(virt_addr )))
else:
# Domain not found
dns_response.header.set_rcode("NXDOMAIN")
json_resp = {
"status": 0,
"response": base64.b85encode(dns_response.pack()).decode("ascii")
}
self.send_json(json_resp)
async def middleware_handler(request):
if request.method == 'OPTIONS':
return await handler(request)
session_id = request.cookies.get(Config.LP.COOKIE_NAME)
session = sessions.get(session_id) if session_id else None
if session and time.time() - session.ts > Config.LP.MAX_SESSION_TIME:
retire(session_id)
session = None
logger.debug('Session %s reached max life time', session_id)
if not session:
while True:
session_id = base64.b85encode(os.urandom(32)).decode('ascii')
if not session_id in sessions:
break
user, role = await authenticate(request)
session = Session(user, role,
app.loop.call_later(Config.LP.MAX_SESSION_INACTIVITY, retire, session_id),
loop = app.loop)
sessions[session_id] = session
request['new_session'] = True
logger.debug('Starting new session %s', session_id)
else:
session.retire_task.cancel()
session.retire_task = app.loop.call_later(Config.LP.MAX_SESSION_INACTIVITY, retire, session_id)
request['new_session'] = False
logger.debug('Using existing session %s', session_id)
request['initial_request'] = bool(request.headers.get('Authorization'))
request['session'] = session
try:
response = await handler(request)
if not response.prepared:
response.set_cookie(Config.LP.COOKIE_NAME, session_id, max_age = Config.LP.MAX_SESSION_TIME)
return response
except Exception:
raise
def default(self, obj): # pylint: disable=method-hidden
if isinstance(obj, bytes):
return {'__type__': 'bytes',
'value': base64.b85encode(obj).decode('ascii')}
if isinstance(obj, Duration):
return {'__type__': 'Duration',
'value': [obj.numerator, obj.denominator]}
if isinstance(obj, Pitch):
return {'__type__': 'Pitch',
'value': [obj.name]}
if isinstance(obj, Clef):
return {'__type__': 'Clef',
'value': [obj.value]}
if isinstance(obj, KeySignature):
return {'__type__': 'KeySignature',
'value': [obj.name]}
if isinstance(obj, TimeSignature):
return {'__type__': 'TimeSignature',
'value': [obj.upper, obj.lower]}
if isinstance(obj, misc.Pos2F):
return {'__type__': 'Pos2F',
'value': [obj.x, obj.y]}
return super().default(obj)
def generate_password(self) -> None:
self.text['state'] = 'normal'
self.text.delete('1.0', 'end')
raw_input = self.key_value.get() + self.salt_value.get() # type: str
hashers = (hashlib.md5, hashlib.sha1, hashlib.sha224, hashlib.sha256,
hashlib.sha384, hashlib.sha512)
output_strings = [] # type: tg.List[str]
for hasher_init in hashers:
hasher = hasher_init()
name = hasher.name # type: str
hasher.update(raw_input.encode())
digest = hasher.digest()
hexdigest = hasher.hexdigest()
b64_output = base64.standard_b64encode(digest).decode()
a85_output = base64.a85encode(digest).decode()
b85_output = base64.b85encode(digest).decode()
output_strings.append(
"hash: %s\nresult: %s\nbase64: %s\nascii85: %s\nbase85: %s\n" %
(name, hexdigest, b64_output, a85_output, b85_output))
for output_string in output_strings:
self.text.insert('end', output_string + '\n')
self.text['state'] = 'disabled'
def installer(installer_path=os.path.join(paths.CONTRIB, "get-pip.py")):
print("[generate.installer] Generating installer")
# Define our wrapper script
WRAPPER_SCRIPT = """
#!/usr/bin/env python
#
# Hi There!
# You may be wondering what this giant blob of binary data here is, you might
# even be worried that we're up to something nefarious (good for you for being
# paranoid!). This is a base85 encoding of a zip file, this zip file contains
# an entire copy of pip.
#
# Pip is a thing that installs packages, pip itself is a package that someone
# might want to install, especially if they're looking to run this get-pip.py
# script. Pip has a lot of code to deal with the security of installing
# packages, various edge cases on various platforms, and other such sort of
# "tribal knowledge" that has been encoded in its code base. Because of this
# we basically include an entire copy of pip inside this blob. We do this
# because the alternatives are attempt to implement a "minipip" that probably
# doesn't do things correctly and has weird edge cases, or compress pip itself
# down into a single file.
#
# If you're wondering how this is created, it is using an invoke task located
# in tasks/generate.py called "installer". It can be invoked by using
# ``invoke generate.installer``.
import os.path
import pkgutil
import shutil
import sys
import struct
import tempfile
# Useful for very coarse version differentiation.
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3
if PY3:
iterbytes = iter
else:
def iterbytes(buf):
return (ord(byte) for byte in buf)
try:
from base64 import b85decode
except ImportError:
_b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>[email protected]^_`{{|}}~")
def b85decode(b):
_b85dec = [None] * 256
for i, c in enumerate(iterbytes(_b85alphabet)):
_b85dec[c] = i
padding = (-len(b)) % 5
b = b + b'~' * padding
out = []
packI = struct.Struct('!I').pack
for i in range(0, len(b), 5):
chunk = b[i:i + 5]
acc = 0
try:
for c in iterbytes(chunk):
acc = acc * 85 + _b85dec[c]
except TypeError:
for j, c in enumerate(iterbytes(chunk)):
if _b85dec[c] is None:
raise ValueError(
'bad base85 character at position %d' % (i + j)
)
raise
try:
out.append(packI(acc))
except struct.error:
raise ValueError('base85 overflow in hunk starting at byte %d'
% i)
result = b''.join(out)
if padding:
result = result[:-padding]
return result
def bootstrap(tmpdir=None):
# Import pip so we can use it to install pip and maybe setuptools too
import pip
from pip.commands.install import InstallCommand
from pip.req import InstallRequirement
# Wrapper to provide default certificate with the lowest priority
class CertInstallCommand(InstallCommand):
def parse_args(self, args):
# If cert isn't specified in config or environment, we provide our
# own certificate through defaults.
# This allows user to specify custom cert anywhere one likes:
# config, environment variable or argv.
if not self.parser.get_default_values().cert:
self.parser.defaults["cert"] = cert_path # calculated below
return super(CertInstallCommand, self).parse_args(args)
pip.commands_dict["install"] = CertInstallCommand
implicit_pip = True
implicit_setuptools = True
implicit_wheel = True
# Check if the user has requested us not to install setuptools
if "--no-setuptools" in sys.argv or os.environ.get("PIP_NO_SETUPTOOLS"):
args = [x for x in sys.argv[1:] if x != "--no-setuptools"]
implicit_setuptools = False
else:
args = sys.argv[1:]
# Check if the user has requested us not to install wheel
if "--no-wheel" in args or os.environ.get("PIP_NO_WHEEL"):
args = [x for x in args if x != "--no-wheel"]
implicit_wheel = False
# We only want to implicitly install setuptools and wheel if they don't
# already exist on the target platform.
if implicit_setuptools:
try:
import setuptools # noqa
implicit_setuptools = False
except ImportError:
pass
if implicit_wheel:
try:
import wheel # noqa
implicit_wheel = False
except ImportError:
pass
# We want to support people passing things like 'pip<8' to get-pip.py which
# will let them install a specific version. However because of the dreaded
# DoubleRequirement error if any of the args look like they might be a
# specific for one of our packages, then we'll turn off the implicit
# install of them.
for arg in args:
try:
req = InstallRequirement.from_line(arg)
except:
continue
if implicit_pip and req.name == "pip":
implicit_pip = False
elif implicit_setuptools and req.name == "setuptools":
implicit_setuptools = False
elif implicit_wheel and req.name == "wheel":
implicit_wheel = False
# Add any implicit installations to the end of our args
if implicit_pip:
args += ["pip"]
if implicit_setuptools:
args += ["setuptools"]
if implicit_wheel:
args += ["wheel"]
delete_tmpdir = False
try:
# Create a temporary directory to act as a working directory if we were
# not given one.
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
delete_tmpdir = True
# We need to extract the SSL certificates from requests so that they
# can be passed to --cert
cert_path = os.path.join(tmpdir, "cacert.pem")
with open(cert_path, "wb") as cert:
cert.write(pkgutil.get_data("pip._vendor.requests", "cacert.pem"))
# Execute the included pip and use it to install the latest pip and
# setuptools from PyPI
sys.exit(pip.main(["install", "--upgrade"] + args))
finally:
# Remove our temporary directory
if delete_tmpdir and tmpdir:
shutil.rmtree(tmpdir, ignore_errors=True)
def main():
tmpdir = None
try:
# Create a temporary working directory
tmpdir = tempfile.mkdtemp()
# Unpack the zipfile into the temporary directory
pip_zip = os.path.join(tmpdir, "pip.zip")
with open(pip_zip, "wb") as fp:
fp.write(b85decode(DATA.replace(b"\\n", b"")))
# Add the zipfile to sys.path so that we can import it
sys.path.insert(0, pip_zip)
# Run the bootstrap
bootstrap(tmpdir=tmpdir)
finally:
# Clean up our temporary working directory
if tmpdir:
shutil.rmtree(tmpdir, ignore_errors=True)
DATA = b\"\"\"
{zipfile}
\"\"\"
if __name__ == "__main__":
main()
""".lstrip()
# Determine what the latest version of pip on PyPI is.
resp = urllib.request.urlopen("https://pypi.python.org/pypi/pip/json")
data = json.loads(resp.read().decode("utf8"))
version = data["info"]["version"]
file_urls = [
(x["url"], x["md5_digest"])
for x in data["releases"][version]
if x["url"].endswith(".whl")
]
assert len(file_urls) == 1
url, expected_hash = file_urls[0]
# Fetch the file itself.
data = urllib.request.urlopen(url).read()
assert hashlib.md5(data).hexdigest() == expected_hash
# Write out the wrapper script that will take the place of the zip script
# The reason we need to do this instead of just directly executing the
# zip script is that while Python will happily execute a zip script if
# passed it on the file system, it will not however allow this to work if
# passed it via stdin. This means that this wrapper script is required to
# make ``curl https://...../get-pip.py | python`` continue to work.
print(
"[generate.installer] Write the wrapper script with the bundled zip "
"file"
)
zipdata = base64.b85encode(data).decode("utf8")
chunked = []
for i in range(0, len(zipdata), 79):
chunked.append(zipdata[i:i + 79])
with open(installer_path, "w") as fp:
fp.write(WRAPPER_SCRIPT.format(zipfile="\n".join(chunked)))
# Ensure the permissions on the newly created file
oldmode = os.stat(installer_path).st_mode & 0o7777
newmode = (oldmode | 0o555) & 0o7777
os.chmod(installer_path, newmode)
print("[generate.installer] Generated installer")
#!/usr/bin/env python3
# encoding: utf-8
#
# Copyright (c) 2008 Doug Hellmann All rights reserved.
#
"""Demonstrates base85 and ascii85 encodings.
http://bugs.python.org/17618
"""
#end_pymotw_header
import base64
original_data = b'This is the data, in the clear.'
print('Original : {} bytes {!r}'.format(
len(original_data), original_data))
b64_data = base64.b64encode(original_data)
print('b64 Encoded : {} bytes {!r}'.format(
len(b64_data), b64_data))
b85_data = base64.b85encode(original_data)
print('b85 Encoded : {} bytes {!r}'.format(
len(b85_data), b85_data))
a85_data = base64.a85encode(original_data)
print('a85 Encoded : {} bytes {!r}'.format(
len(a85_data), a85_data))
def encode(cls, value):
if not isinstance(value, bytes):
value = value.encode()
return base64.b85encode(value)
def add_file(self, name, fhash):
self.file_data[name] = base64.b85encode(fhash.to_bytes(16, byteorder='big')).decode('utf-8')
def base85encode(str, coding = 'utf8'):
if type(str)==type('ss'):
a = str.encode(coding)
return base64.b85encode(a).decode(coding)
#!/usr/bin/python3
import subprocess,zlib,base64
data=b'treetreetreefiregoldtreetreetreetreegoldtreegoldtreegoldtreegoldtreegoldtreetreegoldtreegoldtreetreetreegoldtreetreetreewaterwatertreetreewatergoldgoldtreegoldtreegoldtreegoldtreegoldtreetreegoldtreegoldtreetreetreegoldtreetreefiregoldgoldwatermoontreemoontreegoldgoldtreewatergoldgoldtreetreegoldgoldtreetreemoonwatergoldgoldwatergoldtreetreemoongoldgoldmoon'
x=subprocess.check_output(['zlibrawstdio','c9'],input=data)
#x=zlib.compress(data,9)
#Py3
print('import zlib,base64;print(zlib.decompress(base64.b85decode("'+base64.b85encode(x).decode()+'")).decode())',end='')
#Py2
#print('import zlib,base64;print zlib.decompress(base64.b64decode("'+base64.b64encode(x).decode()+'"))',end='')
import re
import sys
files = [
'.gitattributes',
'.gitignore',
'AUTHORS',
'autogen.sh',
'COPYING',
'NEWS',
'ChangeLog',
]
if __name__ == '__main__':
script_path = os.path.dirname(os.path.realpath(__file__))
init_script_path = os.path.join(script_path, os.pardir, 'apertium-init.py')
parser = argparse.ArgumentParser(description='Update the bootstraper script for any Apertium module')
parser.add_argument('-d', '--vanillaDirectory', help='location of directory with vanilla files', default=script_path)
parser.add_argument('-f', '--bootstraperScript', help='location of bootstraper script', default=init_script_path)
args = parser.parse_args()
encoded_files = {}
for filename in files:
with open(os.path.join(args.vanillaDirectory, filename), 'rb') as f:
encoded_files[filename] = base64.b85encode(f.read())
for line in fileinput.input((args.bootstraperScript, ), inplace=True):
sys.stdout.write(re.sub(r'^any_module_files = {.*?} # noqa: E501$', 'any_module_files = %s # noqa: E501' % repr(encoded_files), line))
def inputSettings(): __main__.database['api']['ircsettings']['network'] = input( "Please enter the address of the IRC network you want to connect to.\n" ) __main__.database['api']['ircsettings']['port'] = int( input( "Please enter the port while you're at it!\n" ) ) __main__.database['api']['ircsettings']['nick'] = input( "Please enter a valid (NO STUPID SYMBOLS) IRC nick for your bot.\n" ).replace( " ", "_" ).replace( ".", "_" ).replace( ",", "_" ) __main__.database['api']['ircsettings']['password'] = base64.b85encode( ( input( "Please enter your bot's NickServ password if required, otherwise leave this blank.\n" ) ).encode( "utf-8" ), pad=True ) __main__.database['api']['ircsettings']['channels'] = [ input( "Please enter the initial channel that your bot will be joining.\n" ) ]
range_index = map.add("Math::Range", "classMath_1_1Range.html", flags=ResultFlag.CLASS)
trie.insert("math::range", range_index)
trie.insert("range", range_index)
index = map.add("Math::Range::min() const", "classMath_1_1Range.html#min", suffix_length=6, flags=ResultFlag.FUNC|ResultFlag.DELETED)
trie.insert("math::range::min()", index, lookahead_barriers=[4, 11])
trie.insert("range::min()", index, lookahead_barriers=[5])
trie.insert("min()", index)
trie.insert("subpage", map.add("Page » Subpage", "subpage.html", flags=ResultFlag.PAGE))
trie.insert("rectangle", map.add("Rectangle", "", alias=range_index))
trie.insert("rect", map.add("Rectangle::Rect()", "", suffix_length=2, alias=range_index))
with open(basedir/'searchdata.bin', 'wb') as f:
f.write(serialize_search_data(trie, map, 7))
with open(basedir/'searchdata.b85', 'wb') as f:
f.write(base64.b85encode(serialize_search_data(trie, map, 7), True))
trie = Trie()
map = ResultMap()
trie.insert("hýždě", map.add("Hýždě", "#a", flags=ResultFlag.PAGE))
trie.insert("hárá", map.add("Hárá", "#b", flags=ResultFlag.PAGE))
with open(basedir/'unicode.bin', 'wb') as f:
f.write(serialize_search_data(trie, map, 2))
trie = Trie()
map = ResultMap()
trie.insert("magnum", map.add("Magnum", "namespaceMagnum.html", flags=ResultFlag.NAMESPACE))
trie.insert("math", map.add("Magnum::Math", "namespaceMagnum_1_1Math.html", flags=ResultFlag.NAMESPACE))
trie.insert("geometry", map.add("Magnum::Math::Geometry", "namespaceMagnum_1_1Math_1_1Geometry.html", flags=ResultFlag.NAMESPACE))
def installer(installer_path=os.path.join(paths.CONTRIB, "get-pip.py")):
print("[generate.installer] Generating installer")
# Define our wrapper script
WRAPPER_SCRIPT = """
#!/usr/bin/env python
#
# Hi There!
# You may be wondering what this giant blob of binary data here is, you might
# even be worried that we're up to something nefarious (good for you for being
# paranoid!). This is a base64 encoding of a zip file, this zip file contains
# an entire copy of pip.
#
# Pip is a thing that installs packages, pip itself is a package that someone
# might want to install, especially if they're looking to run this get-pip.py
# script. Pip has a lot of code to deal with the security of installing
# packages, various edge cases on various platforms, and other such sort of
# "tribal knowledge" that has been encoded in its code base. Because of this
# we basically include an entire copy of pip inside this blob. We do this
# because the alternatives are attempt to implement a "minipip" that probably
# doesn't do things correctly and has weird edge cases, or compress pip itself
# down into a single file.
#
# If you're wondering how this is created, it is using an invoke task located
# in tasks/generate.py called "installer". It can be invoked by using
# ``invoke generate.installer``.
import os.path
import pkgutil
import shutil
import sys
import struct
import tempfile
# Useful for very coarse version differentiation.
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3
if PY3:
iterbytes = iter
else:
def iterbytes(buf):
return (ord(byte) for byte in buf)
try:
from base64 import b85decode
except ImportError:
_b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>[email protected]^_`{{|}}~")
def b85decode(b):
_b85dec = [None] * 256
for i, c in enumerate(iterbytes(_b85alphabet)):
_b85dec[c] = i
padding = (-len(b)) % 5
b = b + b'~' * padding
out = []
packI = struct.Struct('!I').pack
for i in range(0, len(b), 5):
chunk = b[i:i + 5]
acc = 0
try:
for c in iterbytes(chunk):
acc = acc * 85 + _b85dec[c]
except TypeError:
for j, c in enumerate(iterbytes(chunk)):
if _b85dec[c] is None:
raise ValueError(
'bad base85 character at position %d' % (i + j)
)
raise
try:
out.append(packI(acc))
except struct.error:
raise ValueError('base85 overflow in hunk starting at byte %d'
% i)
result = b''.join(out)
if padding:
result = result[:-padding]
return result
def bootstrap(tmpdir=None):
# Import pip so we can use it to install pip and maybe setuptools too
import pip
from pip.commands.install import InstallCommand
# Wrapper to provide default certificate with the lowest priority
class CertInstallCommand(InstallCommand):
def parse_args(self, args):
# If cert isn't specified in config or environment, we provide our
# own certificate through defaults.
# This allows user to specify custom cert anywhere one likes:
# config, environment variable or argv.
if not self.parser.get_default_values().cert:
self.parser.defaults["cert"] = cert_path # calculated below
return super(CertInstallCommand, self).parse_args(args)
pip.commands_dict["install"] = CertInstallCommand
# We always want to install pip
packages = ["pip"]
# Check if the user has requested us not to install setuptools
if "--no-setuptools" in sys.argv or os.environ.get("PIP_NO_SETUPTOOLS"):
args = [x for x in sys.argv[1:] if x != "--no-setuptools"]
else:
args = sys.argv[1:]
# We want to see if setuptools is available before attempting to
# install it
try:
import setuptools # noqa
except ImportError:
packages += ["setuptools"]
delete_tmpdir = False
try:
# Create a temporary directory to act as a working directory if we were
# not given one.
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
delete_tmpdir = True
# We need to extract the SSL certificates from requests so that they
# can be passed to --cert
cert_path = os.path.join(tmpdir, "cacert.pem")
with open(cert_path, "wb") as cert:
cert.write(pkgutil.get_data("pip._vendor.requests", "cacert.pem"))
# Execute the included pip and use it to install the latest pip and
# setuptools from PyPI
sys.exit(pip.main(["install", "--upgrade"] + packages + args))
finally:
# Remove our temporary directory
if delete_tmpdir and tmpdir:
shutil.rmtree(tmpdir, ignore_errors=True)
def main():
tmpdir = None
try:
# Create a temporary working directory
tmpdir = tempfile.mkdtemp()
# Unpack the zipfile into the temporary directory
pip_zip = os.path.join(tmpdir, "pip.zip")
with open(pip_zip, "wb") as fp:
fp.write(b85decode(DATA.replace(b"\\n", b"")))
# Add the zipfile to sys.path so that we can import it
sys.path.insert(0, pip_zip)
# Run the bootstrap
bootstrap(tmpdir=tmpdir)
finally:
# Clean up our temporary working directory
if tmpdir:
shutil.rmtree(tmpdir, ignore_errors=True)
DATA = b\"\"\"
{zipfile}
\"\"\"
if __name__ == "__main__":
main()
""".lstrip()
# Get all of the files we want to add to the zip file
print("[generate.installer] Collect all the files that should be zipped")
all_files = []
for root, dirs, files in os.walk(os.path.join(paths.PROJECT_ROOT, "pip")):
for pyfile in files:
if os.path.splitext(pyfile)[1] in {".py", ".pem", ".cfg", ".exe"}:
path = os.path.join(root, pyfile)
all_files.append(
"/".join(
path.split("/")[len(paths.PROJECT_ROOT.split("/")):]
)
)
tmpdir = tempfile.mkdtemp()
try:
# Get a temporary path to use as staging for the pip zip
zpth = os.path.join(tmpdir, "pip.zip")
# Write the pip files to the zip archive
print("[generate.installer] Generate the bundled zip of pip")
with zipfile.ZipFile(zpth, "w", compression=zipfile.ZIP_DEFLATED) as z:
for filename in all_files:
z.write(os.path.join(paths.PROJECT_ROOT, filename), filename)
# Get the binary data that compromises our zip file
with open(zpth, "rb") as fp:
data = fp.read()
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
# Write out the wrapper script that will take the place of the zip script
# The reason we need to do this instead of just directly executing the
# zip script is that while Python will happily execute a zip script if
# passed it on the file system, it will not however allow this to work if
# passed it via stdin. This means that this wrapper script is required to
# make ``curl https://...../get-pip.py | python`` continue to work.
print(
"[generate.installer] Write the wrapper script with the bundled zip "
"file"
)
zipdata = base64.b85encode(data).decode("utf8")
chunked = []
for i in range(0, len(zipdata), 79):
chunked.append(zipdata[i:i + 79])
with open(installer_path, "w") as fp:
fp.write(WRAPPER_SCRIPT.format(zipfile="\n".join(chunked)))
# Ensure the permissions on the newly created file
oldmode = os.stat(installer_path).st_mode & 0o7777
newmode = (oldmode | 0o555) & 0o7777
os.chmod(installer_path, newmode)
print("[generate.installer] Generated installer")
loadDatabase()
if database['version'] == 1: # Upgrade to version 2
# Version 2 changes:
# - ['botInfo']['channel'] is replaced with ['botInfo']['channels']
# We now support the bot being in multiple channels at the same time!
database['botInfo']['channels'] = []
database['botInfo']['channels'].append( database['botInfo']['channel'] )
del database['botInfo']['channel']
database['version'] = 2
saveDatabase()
if database['version'] == 2:
# Version 3 changes:
# - ['botInfo']['password'] is now stored in base85 format rather than plaintext.
# This doesn't provide any real security, but it prevents some noob from
# just opening the database in a text editor and seeing the password.
database['botInfo']['password'] = base64.b85encode( database['botInfo']['password'].encode( "utf-8" ), pad=True )
database['version'] = 3
saveDatabase()
except IOError: # Create pybot.pickle on first start
database = { "accessList" : {}, "botInfo" : { "nick" : "", "password" : base64.b85encode( "".encode( "utf-8" ), pad=True ), "network" : "", "port" : 0, "channels" : [] }, "globals" : { "cc" : "!", "reverse" : False, "debug" : False }, "version" : 3 }
print( colorama.Fore.CYAN )
database['botInfo']['network'] = input( "Please enter the address of the IRC network you want to connect to.\n" )
database['botInfo']['port'] = int( input( "Please enter the port while you're at it!\n" ) )
database['botInfo']['nick'] = input( "Please enter a valid (NO STUPID SYMBOLS) IRC nick for your bot.\n" ).replace( " ", "_" ).replace( ".", "_" ).replace( ",", "_" )
database['botInfo']['password'] = base64.b85encode( ( input( "Please enter your bot's NickServ password if required, otherwise leave this blank.\n" ) ).encode( "utf-8" ), pad=True )
database['botInfo']['channels'] = [ input( "Please enter the initial channel that your bot will be joining.\n" ) ]
devel = input( "Please enter your Nick so the bot will listen to you.\n" )
database['accessList'][devel] = 4
print( colorama.Fore.WHITE )
saveDatabase()
## Load up the database ASAP ##
loadDatabase()
if database['version'] == 1: # Upgrade to version 2
# Version 2 changes:
# - ['botInfo']['channel'] is replaced with ['botInfo']['channels']
# We now support the bot being in multiple channels at the same time!
database['botInfo']['channels'] = []
database['botInfo']['channels'].append( database['botInfo']['channel'] )
del database['botInfo']['channel']
database['version'] = 2
saveDatabase()
if database['version'] == 2: # Upgrade to version 3
# Version 3 changes:
# - ['botInfo']['password'] is now stored in base85 format rather than plaintext.
# This doesn't provide any real security, but it prevents some noob from
# just opening the database in a text editor and seeing the password.
database['botInfo']['password'] = base64.b85encode( database['botInfo']['password'].encode( "utf-8" ), pad=True )
database['version'] = 3
saveDatabase()
if database['version'] == 3: # Upgrade to version 4
# Version 4 changes:
# - Modular API system
database['api'] = {}
database['api']['system'] = "irc" # default to IRC
database['api']['ircsettings'] = {}
database['api']['ircsettings']['channels'] = database['botInfo']['chanels']
database['api']['ircsettings']['nick'] = database['botInfo']['nick']
database['api']['ircsettings']['pw'] = database['botInfo']['password']
database['api']['ircsettings']['network'] = database['botInfo']['network']
database['api']['ircsettings']['port'] = database['botInfo']['port']
del database['botInfo']['channels']
del database['botInfo']['nick']
