def filter_clean():
# Operate on stdin/stdout in binary mode
firstline = next(stdin)
if firstline == b"bigstore\n":
stdout.write(firstline)
for line in stdin:
stdout.write(line)
else:
file = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
hash_function = default_hash_function()
hash_function.update(firstline)
file.write(firstline)
for line in stdin:
hash_function.update(line)
file.write(line)
file.close()
hexdigest = hash_function.hexdigest()
mkdir_p(os.path.join(object_directory(default_hash_function_name), hexdigest[:2]))
shutil.copy(file.name, object_filename(default_hash_function_name, hexdigest))
stdout.write(b"bigstore\n")
stdout.write(native_str_to_bytes("{}\n".format(default_hash_function_name)))
stdout.write(native_str_to_bytes("{}\n".format(hexdigest)))
def filter_clean():
# Operate on stdin/stdout in binary mode
firstline = next(stdin)
if firstline == b"bigstore\n":
stdout.write(firstline)
for line in stdin:
stdout.write(line)
else:
file = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
hash_function = default_hash_function()
hash_function.update(firstline)
file.write(firstline)
for line in stdin:
hash_function.update(line)
file.write(line)
file.close()
hexdigest = hash_function.hexdigest()
mkdir_p(
os.path.join(object_directory(default_hash_function_name),
hexdigest[:2]))
shutil.copy(file.name,
object_filename(default_hash_function_name, hexdigest))
stdout.write(b"bigstore\n")
stdout.write(
native_str_to_bytes("{}\n".format(default_hash_function_name)))
stdout.write(native_str_to_bytes("{}\n".format(hexdigest)))
def convert_to_bytes(cls, elem):
if isinstance(elem, str):
elem = native_str_to_bytes(elem)
elif isinstance(elem, list):
elem = [cls.convert_to_bytes(e) for e in elem]
elif isinstance(elem, dict):
elem = OrderedDict(
(native_str_to_bytes(n), cls.convert_to_bytes(e))
for n, e in elem.items())
return elem
def convert_to_bytes(cls, elem):
if isinstance(elem, str):
elem = native_str_to_bytes(elem)
elif isinstance(elem, list):
elem = [cls.convert_to_bytes(e) for e in elem]
elif isinstance(elem, dict):
elem = OrderedDict(
(native_str_to_bytes(n), cls.convert_to_bytes(e))
for n, e in elem.items())
return elem
def get_github_api_for_repo(keychain, owner, repo):
gh = GitHub()
# Apply retry policy
gh.session.mount("http://", adapter)
gh.session.mount("https://", adapter)
APP_KEY = native_str_to_bytes(os.environ.get("GITHUB_APP_KEY", ""))
APP_ID = os.environ.get("GITHUB_APP_ID")
if APP_ID and APP_KEY:
installation = INSTALLATIONS.get((owner, repo))
if installation is None:
gh.login_as_app(APP_KEY, APP_ID, expire_in=120)
try:
installation = gh.app_installation_for_repository(owner, repo)
except github3.exceptions.NotFoundError:
raise GithubException(
"Could not access {}/{} using GitHub app. "
"Does the app need to be installed for this repository?".
format(owner, repo))
INSTALLATIONS[(owner, repo)] = installation
gh.login_as_app_installation(APP_KEY, APP_ID, installation.id)
else:
github_config = keychain.get_service("github")
gh.login(github_config.username, github_config.password)
return gh
def _finalise_dict(self, elem_dict, nineml_type=None, **options):
if nineml_type is None:
nineml_type = Document.nineml_type
nineml_type = native_str_to_bytes(nineml_type)
elem_dict = self.from_elem(elem_dict, nineml_type=nineml_type,
**options)
if PY3:
elem_dict = self.convert_from_bytes(elem_dict)
return elem_dict
def test_backwards_compatibility(self):
full_v1_xml = etree.fromstring(native_str_to_bytes(version1))
full_v2_xml = etree.fromstring(native_str_to_bytes(version2))
v1_doc = XMLUnserializer(root=etree.fromstring(
native_str_to_bytes(version1))).unserialize()
v2_doc = XMLUnserializer(root=etree.fromstring(
native_str_to_bytes(version2))).unserialize()
v1_doc._url = './dummy.xml'
v2_doc._url = './dummy.xml'
# Ensure all elements are loaded
list(v1_doc.elements)
list(v2_doc.elements)
v1_names = list(v1_doc.nineml_types)
v2_names = list(v1_doc.nineml_types)
self.assertEqual(v1_names, v2_names)
for name in v1_names:
v1 = v1_doc[name]
v2 = v2_doc[name]
# Test loaded python objects are equivalent between versions
self.assertEqual(
v1, v2,
"Loaded version 1 didn't match loaded version 2:\n{}".format(
v1.find_mismatch(v2)))
v1_to_v2_xml = XMLSerializer(document=v2_doc,
version=2.0).visit(v1,
ref_style='force')
v2_to_v1_xml = XMLSerializer(document=v1_doc,
version=1.0).visit(v2,
ref_style='force')
v1_xml = self._get_xml_element(full_v1_xml, name)
v2_xml = self._get_xml_element(full_v2_xml, name)
# Test the version 1 converted to version 2
self.assert_(
xml_equal(v1_to_v2_xml, v2_xml),
"v2 produced from v1 doesn't match loaded:\n{}\n\nand"
"\n\n{}".format(xml_to_str(v1_to_v2_xml, pp=True),
xml_to_str(v2_xml, pp=True)))
# Test the version 2 converted to version 1
self.assert_(
xml_equal(v2_to_v1_xml, v1_xml),
"v1 produced from v2 doesn't match loaded:\n{}\n\nand\n\n"
"{}".format(xml_to_str(v2_to_v1_xml, pp=True),
xml_to_str(v1_xml, pp=True)))
def _finalise_dict(self, elem_dict, nineml_type=None, **options):
if nineml_type is None:
nineml_type = Document.nineml_type
nineml_type = native_str_to_bytes(nineml_type)
elem_dict = self.from_elem(elem_dict,
nineml_type=nineml_type,
**options)
if PY3:
elem_dict = self.convert_from_bytes(elem_dict)
return elem_dict
def get_client_resource(self, configuration):
"""
:return: A static data resource that produces the given configuration when
rendered, as an aid to testing.
"""
items = configuration.items(self._client_section_name, [])
resource = Data(
native_str_to_bytes(dumps(dict(items))),
ensure_str("text/json"),
)
# Give it some dynamic stuff too.
resource.putChild(b"counter", GetCounter())
return resource
def on_message(self, msg):
try:
if getattr(self, 'socket', None):
log.info("closing old socket")
self.loop.add_callback(self.do_close)
return
if msg.startswith('discussion:') and self.valid:
self.discussion = msg.split(':', 1)[1]
if msg.startswith('token:') and self.valid:
try:
self.raw_token = msg.split(':', 1)[1]
self.token = decode_token(self.raw_token, TOKEN_SECRET)
if self.token['userId'] != Everyone:
self.userId = 'local:Agent/' + str(
self.token['userId'])
else:
self.userId = Everyone
except TokenInvalid:
pass
if self.token and self.discussion:
# Check if token authorizes discussion
r = requests.get(
'%s/api/v1/discussion/%s/permissions/read/u/%s' %
(SERVER_URL, self.discussion, self.token['userId']),
headers={"Accept": "application/json"})
log.debug(r.text)
if r.text != 'true':
return
self.socket = context.socket(zmq.SUB)
self.socket.connect(INTERNAL_SOCKET)
self.socket.setsockopt(zmq.SUBSCRIBE, b'*')
self.socket.setsockopt(
zmq.SUBSCRIBE, native_str_to_bytes(str(self.discussion)))
self.loop = zmqstream.ZMQStream(self.socket, io_loop=io_loop)
self.loop.on_recv(self.on_recv)
log.info("connected")
self.send('[{"@type":"Connection"}]')
if self.token and self.raw_token and self.discussion and self.userId != Everyone:
requests.post(
'%s/data/Discussion/%s/all_users/%d/connecting' %
(SERVER_URL, self.discussion, self.token['userId']),
data={'token': self.raw_token})
except Exception:
capture_exception()
self.do_close()
def flatten_source_post_id(cls, source_post_id, extra_length=0):
# Ensure that a source_post_id can be used as part 1 of message_id
sanitized = cls._non_email_chars.subn(
lambda c: '_' + hex(ord(c.group()))[2:], source_post_id)[0]
if len(sanitized) + extra_length > 64:
# 64 is max according to RFC 5322
# cut it short and add a digest of original
import hashlib
import base64
d = hashlib.md5()
d.update(native_str_to_bytes(source_post_id, 'utf-8'))
d = bytes_to_native_str(base64.urlsafe_b64encode(d.digest()))
sanitized = sanitized[:max(0, 64 - len(d) - extra_length - 1)]
if sanitized:
sanitized += "_" + d
else:
sanitized = d
return sanitized
def json_rpc_dumps(o):
return native_str_to_bytes(
json.dumps(o, default=_json_default, ensure_ascii=True))
def render_GET(self, request):
self.value += 1
return native_str_to_bytes(dumps({"value": self.value}))
def json_rpc_dumps(o):
return native_str_to_bytes(json.dumps(o, default=_json_default,
ensure_ascii=True))
def from_str(self, string, **options): # @UnusedVariable
try:
return etree.fromstring(native_str_to_bytes(string))
except etree.LxmlError as e:
raise NineMLSerializationError(
"Could not parse XML string '{}': \n{}".format(string, e))
def from_str(self, string, **options): # @UnusedVariable
try:
return etree.fromstring(native_str_to_bytes(string))
except etree.LxmlError as e:
raise NineMLSerializationError(
"Could not parse XML string '{}': \n{}".format(string, e))