def filter_clean(): # Operate on stdin/stdout in binary mode firstline = next(stdin) if firstline == b"bigstore\n": stdout.write(firstline) for line in stdin: stdout.write(line) else: file = tempfile.NamedTemporaryFile(mode='w+b', delete=False) hash_function = default_hash_function() hash_function.update(firstline) file.write(firstline) for line in stdin: hash_function.update(line) file.write(line) file.close() hexdigest = hash_function.hexdigest() mkdir_p(os.path.join(object_directory(default_hash_function_name), hexdigest[:2])) shutil.copy(file.name, object_filename(default_hash_function_name, hexdigest)) stdout.write(b"bigstore\n") stdout.write(native_str_to_bytes("{}\n".format(default_hash_function_name))) stdout.write(native_str_to_bytes("{}\n".format(hexdigest)))
def filter_clean(): # Operate on stdin/stdout in binary mode firstline = next(stdin) if firstline == b"bigstore\n": stdout.write(firstline) for line in stdin: stdout.write(line) else: file = tempfile.NamedTemporaryFile(mode='w+b', delete=False) hash_function = default_hash_function() hash_function.update(firstline) file.write(firstline) for line in stdin: hash_function.update(line) file.write(line) file.close() hexdigest = hash_function.hexdigest() mkdir_p( os.path.join(object_directory(default_hash_function_name), hexdigest[:2])) shutil.copy(file.name, object_filename(default_hash_function_name, hexdigest)) stdout.write(b"bigstore\n") stdout.write( native_str_to_bytes("{}\n".format(default_hash_function_name))) stdout.write(native_str_to_bytes("{}\n".format(hexdigest)))
def convert_to_bytes(cls, elem): if isinstance(elem, str): elem = native_str_to_bytes(elem) elif isinstance(elem, list): elem = [cls.convert_to_bytes(e) for e in elem] elif isinstance(elem, dict): elem = OrderedDict( (native_str_to_bytes(n), cls.convert_to_bytes(e)) for n, e in elem.items()) return elem
def convert_to_bytes(cls, elem): if isinstance(elem, str): elem = native_str_to_bytes(elem) elif isinstance(elem, list): elem = [cls.convert_to_bytes(e) for e in elem] elif isinstance(elem, dict): elem = OrderedDict( (native_str_to_bytes(n), cls.convert_to_bytes(e)) for n, e in elem.items()) return elem
def get_github_api_for_repo(keychain, owner, repo): gh = GitHub() # Apply retry policy gh.session.mount("http://", adapter) gh.session.mount("https://", adapter) APP_KEY = native_str_to_bytes(os.environ.get("GITHUB_APP_KEY", "")) APP_ID = os.environ.get("GITHUB_APP_ID") if APP_ID and APP_KEY: installation = INSTALLATIONS.get((owner, repo)) if installation is None: gh.login_as_app(APP_KEY, APP_ID, expire_in=120) try: installation = gh.app_installation_for_repository(owner, repo) except github3.exceptions.NotFoundError: raise GithubException( "Could not access {}/{} using GitHub app. " "Does the app need to be installed for this repository?". format(owner, repo)) INSTALLATIONS[(owner, repo)] = installation gh.login_as_app_installation(APP_KEY, APP_ID, installation.id) else: github_config = keychain.get_service("github") gh.login(github_config.username, github_config.password) return gh
def _finalise_dict(self, elem_dict, nineml_type=None, **options): if nineml_type is None: nineml_type = Document.nineml_type nineml_type = native_str_to_bytes(nineml_type) elem_dict = self.from_elem(elem_dict, nineml_type=nineml_type, **options) if PY3: elem_dict = self.convert_from_bytes(elem_dict) return elem_dict
def test_backwards_compatibility(self): full_v1_xml = etree.fromstring(native_str_to_bytes(version1)) full_v2_xml = etree.fromstring(native_str_to_bytes(version2)) v1_doc = XMLUnserializer(root=etree.fromstring( native_str_to_bytes(version1))).unserialize() v2_doc = XMLUnserializer(root=etree.fromstring( native_str_to_bytes(version2))).unserialize() v1_doc._url = './dummy.xml' v2_doc._url = './dummy.xml' # Ensure all elements are loaded list(v1_doc.elements) list(v2_doc.elements) v1_names = list(v1_doc.nineml_types) v2_names = list(v1_doc.nineml_types) self.assertEqual(v1_names, v2_names) for name in v1_names: v1 = v1_doc[name] v2 = v2_doc[name] # Test loaded python objects are equivalent between versions self.assertEqual( v1, v2, "Loaded version 1 didn't match loaded version 2:\n{}".format( v1.find_mismatch(v2))) v1_to_v2_xml = XMLSerializer(document=v2_doc, version=2.0).visit(v1, ref_style='force') v2_to_v1_xml = XMLSerializer(document=v1_doc, version=1.0).visit(v2, ref_style='force') v1_xml = self._get_xml_element(full_v1_xml, name) v2_xml = self._get_xml_element(full_v2_xml, name) # Test the version 1 converted to version 2 self.assert_( xml_equal(v1_to_v2_xml, v2_xml), "v2 produced from v1 doesn't match loaded:\n{}\n\nand" "\n\n{}".format(xml_to_str(v1_to_v2_xml, pp=True), xml_to_str(v2_xml, pp=True))) # Test the version 2 converted to version 1 self.assert_( xml_equal(v2_to_v1_xml, v1_xml), "v1 produced from v2 doesn't match loaded:\n{}\n\nand\n\n" "{}".format(xml_to_str(v2_to_v1_xml, pp=True), xml_to_str(v1_xml, pp=True)))
def _finalise_dict(self, elem_dict, nineml_type=None, **options): if nineml_type is None: nineml_type = Document.nineml_type nineml_type = native_str_to_bytes(nineml_type) elem_dict = self.from_elem(elem_dict, nineml_type=nineml_type, **options) if PY3: elem_dict = self.convert_from_bytes(elem_dict) return elem_dict
def get_client_resource(self, configuration): """ :return: A static data resource that produces the given configuration when rendered, as an aid to testing. """ items = configuration.items(self._client_section_name, []) resource = Data( native_str_to_bytes(dumps(dict(items))), ensure_str("text/json"), ) # Give it some dynamic stuff too. resource.putChild(b"counter", GetCounter()) return resource
def on_message(self, msg): try: if getattr(self, 'socket', None): log.info("closing old socket") self.loop.add_callback(self.do_close) return if msg.startswith('discussion:') and self.valid: self.discussion = msg.split(':', 1)[1] if msg.startswith('token:') and self.valid: try: self.raw_token = msg.split(':', 1)[1] self.token = decode_token(self.raw_token, TOKEN_SECRET) if self.token['userId'] != Everyone: self.userId = 'local:Agent/' + str( self.token['userId']) else: self.userId = Everyone except TokenInvalid: pass if self.token and self.discussion: # Check if token authorizes discussion r = requests.get( '%s/api/v1/discussion/%s/permissions/read/u/%s' % (SERVER_URL, self.discussion, self.token['userId']), headers={"Accept": "application/json"}) log.debug(r.text) if r.text != 'true': return self.socket = context.socket(zmq.SUB) self.socket.connect(INTERNAL_SOCKET) self.socket.setsockopt(zmq.SUBSCRIBE, b'*') self.socket.setsockopt( zmq.SUBSCRIBE, native_str_to_bytes(str(self.discussion))) self.loop = zmqstream.ZMQStream(self.socket, io_loop=io_loop) self.loop.on_recv(self.on_recv) log.info("connected") self.send('[{"@type":"Connection"}]') if self.token and self.raw_token and self.discussion and self.userId != Everyone: requests.post( '%s/data/Discussion/%s/all_users/%d/connecting' % (SERVER_URL, self.discussion, self.token['userId']), data={'token': self.raw_token}) except Exception: capture_exception() self.do_close()
def flatten_source_post_id(cls, source_post_id, extra_length=0): # Ensure that a source_post_id can be used as part 1 of message_id sanitized = cls._non_email_chars.subn( lambda c: '_' + hex(ord(c.group()))[2:], source_post_id)[0] if len(sanitized) + extra_length > 64: # 64 is max according to RFC 5322 # cut it short and add a digest of original import hashlib import base64 d = hashlib.md5() d.update(native_str_to_bytes(source_post_id, 'utf-8')) d = bytes_to_native_str(base64.urlsafe_b64encode(d.digest())) sanitized = sanitized[:max(0, 64 - len(d) - extra_length - 1)] if sanitized: sanitized += "_" + d else: sanitized = d return sanitized
def json_rpc_dumps(o): return native_str_to_bytes( json.dumps(o, default=_json_default, ensure_ascii=True))
def render_GET(self, request): self.value += 1 return native_str_to_bytes(dumps({"value": self.value}))
def json_rpc_dumps(o): return native_str_to_bytes(json.dumps(o, default=_json_default, ensure_ascii=True))
def from_str(self, string, **options): # @UnusedVariable try: return etree.fromstring(native_str_to_bytes(string)) except etree.LxmlError as e: raise NineMLSerializationError( "Could not parse XML string '{}': \n{}".format(string, e))
def from_str(self, string, **options): # @UnusedVariable try: return etree.fromstring(native_str_to_bytes(string)) except etree.LxmlError as e: raise NineMLSerializationError( "Could not parse XML string '{}': \n{}".format(string, e))