def set(self, content): if isinstance(content, Body): self.mimetype = content.mimetype self.data = content.data self.chunked = content.chunked self.trailer = content.trailer self.fd = content.fd return self.data = None if not content: content = BytesIO() elif isinstance(content, BytesIO) or (hasattr(content, 'read') and hasattr(content, 'fileno') and hasattr(content, 'closed')): if content.closed: raise ValueError('I/O operation on closed file.') elif isinstance(content, Unicode): content = BytesIO(content.encode(self.encoding)) elif isinstance(content, bytes): content = BytesIO(content) elif isinstance(content, bytearray): content = BytesIO(bytes(content)) elif not hasattr(content, '__iter__'): raise TypeError('Content must be iterable.') self.fd = content
async def fivezig_animated(self, ctx, player: MCPlayer): """Get 5zig animated cape by nickname""" uuid = player.uuid try: async with self.session.get( f"http://textures.5zig.net/textures/2/{uuid}", raise_for_status=True) as data: response_data = await data.json(content_type=None, loads=json.loads) if "animatedCape" not in response_data: await ctx.send( chat.error(_("{} doesn't have animated 5zig cape")).format( player.name)) return cape = response_data["animatedCape"] except aiohttp.ClientResponseError as e: if e.status == 404: await ctx.send( chat.error( _("{} doesn't have 5zig cape").format(player.name))) else: await ctx.send( chat.error( _("Unable to get {player}'s 5zig cape: {message}"). format(player=player.name, message=e.message))) return cape = BytesIO(base64.decodebytes(cape.encode())) file = discord.File(cape, filename="{}.png".format(player)) await ctx.send(file=file) cape.close()
def testParseHeaderV1NoBlankLineBetweenHeaderAndBody(self): """ Although it breaks the OFX v1 spec, we don't require an empty line between header and message body """ header = str(self.headerClass(self.defaultVersion)) ofx = header.strip() + "\r\n" + self.body ofx = BytesIO(ofx.encode("ascii")) ofxheader, body = ofxtools.header.parse_header(ofx) self.assertEqual(ofxheader.ofxheader, 100) self.assertEqual(ofxheader.data, "OFXSGML") self.assertEqual(ofxheader.version, self.defaultVersion) self.assertEqual(ofxheader.security, "NONE") self.assertEqual(ofxheader.encoding, "USASCII") self.assertEqual(ofxheader.charset, "NONE") self.assertEqual(ofxheader.compression, "NONE") self.assertEqual(ofxheader.oldfileuid, "NONE") self.assertEqual(ofxheader.newfileuid, "NONE") self.assertEqual(body, self.body) # Make sure body can actually be parsed builder = ofxtools.Parser.TreeBuilder() builder.feed(body) root = builder.close() self.assertIsInstance(root, Element) self.assertEqual(root.tag, "OFX") self.assertIsNone(root.text) self.assertEqual(len(root), 1)
def put(self, path, data, metadata=None): """Store a File object, stream, unicode string, or byte string. Optionally attach metadata to it. """ # File objects with no encoding attributes are byte data. byte_data = not getattr(data, 'encoding', None) # Coerce strings to in-memory Byte streams. if isinstance(data, string_types): data = BytesIO(data.encode()) byte_data = True # Coerce python 3 explicit byte strings if isinstance(data, bytes): data = BytesIO(data) byte_data = True # io.StringIO instances are always unicode if isinstance(data, StringIO): byte_data = False directory = os.path.dirname(os.path.join(self.root, path)) if not os.path.isdir(directory): os.makedirs(directory) fn = os.path.join(self.root, path) if metadata is not None: meta_fn = fn + '.meta' with open(meta_fn, 'w') as f: json.dump(metadata, f) with open(fn, 'wb' if byte_data else 'w') as f: shutil.copyfileobj(data, f)
def testExtraWhitespaceHeaderDemarc(self): # Even though it breaks with the OFX spec, some FIs insert whitespace # after the colon demarc between an OFX header field and its value. # We allow this. header = ("OFXHEADER: 100\r\n" "DATA: OFXSGML\r\n" "VERSION: 103\r\n" "SECURITY: NONE\r\n" "ENCODING: USASCII\r\n" "CHARSET: NONE\r\n" "COMPRESSION: NONE\r\n" "OLDFILEUID: NONE\r\n" "NEWFILEUID: NONE\r\n") ofx = header + self.body ofx = BytesIO(ofx.encode("utf8")) ofxheader, body = ofxtools.header.parse_header(ofx) self.assertEqual(ofxheader.ofxheader, 100) self.assertEqual(ofxheader.data, "OFXSGML") self.assertEqual(ofxheader.version, 103) self.assertEqual(ofxheader.security, "NONE") self.assertEqual(ofxheader.encoding, "USASCII") self.assertEqual(ofxheader.charset, "NONE") self.assertEqual(ofxheader.compression, "NONE") self.assertEqual(ofxheader.oldfileuid, "NONE") self.assertEqual(ofxheader.newfileuid, "NONE") self.assertEqual(body, self.body)
def Read1232IndFile(FileName=None): # ALS beamline 12.3.2 using the XMAS software which will return an ind file containing the intensity of each peak found after indexing a crystal. Also has hkl, two theta, etc. This can be used # for training our algorithms. # Load the ind file with open(FileName, 'r') as f: WholeFile = f.read() # Pull out the chunk of the file that contains the indexation table (starts with number of indexed reflections and ends with the deviatoric stuff. s = re.search('(number of indexed reflections:\s*[0-9]*[\r\n]*)([a-zA-Z0-9\s\(\)_\.-]*)( dev1, dev2)', WholeFile) ReflStr = s.group(2) # Nobu has a typo in his files, so two columns are accidentally blended together. Fix it. ReflStr = ReflStr.replace('chi(deg)intensity', 'chi(deg) intensity') # Now turn the text into a named numpy array. ReflStr = BytesIO(ReflStr.encode('utf-8')) R = np.genfromtxt(ReflStr, names=True) # Sort by hkl R.sort(order=('h', 'k', 'l')) # print(tabulate(R, headers=R.dtype.names)) Rpd = pd.DataFrame(R) Rpd.to_csv(FileName + '.csv') #Rpd.head() return Rpd
def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) # Default returns 200 and the same body message status = 200 body = request.body or "No Body" # check headers are present if self.__checkHeaders(request.headers): # Check signature is valid timestamp = request.headers[self._protocol.TIMESTAMP_HEADER] signature = self._protocol.signRequest(request.url, request.method, params=None, body=request.body, timestamp=timestamp) signatureHeaders = self._protocol.getHeaders(signature) for name, value in signatureHeaders.items(): _v = request.headers.get(name) if not _v or _v != value: status = 403 body = json.dumps({"error" : {"code" : 403, "message": "Headers {} is not correct".format(name)}}) break else: status = 403 body = json.dumps({"error" : {"code" : 403, "message": "Headers are missing"}}) headers = urllib3.response.HTTPHeaderDict(request.headers) if hasattr(body, 'read'): body.seek(0) else: body = BytesIO(body.encode('utf8')) response = urllib3.HTTPResponse(body, headers, status, preload_content=False) return self.build_response(request, response)
def xml_is_valid(xml, only_front=False): xml = BytesIO(xml.encode('utf-8')) try: xml_doc = etree.parse(xml) logger.debug('XML is well formed') except Exception as e: logger.exception(e) logger.error('Fail to parse XML') return (False, '', str(e)) xml_doc = setup_depositor(xml_doc) if only_front: citation_list = xml_doc.find( '//{http://www.crossref.org/schema/4.4.0}citation_list') if citation_list: citation_list.getparent().remove(citation_list) xml_doc_pprint = etree.tostring(xml_doc, pretty_print=True) xml_doc = etree.parse(BytesIO(xml_doc_pprint)) try: result = PARSED_SCHEMA.assertValid(xml_doc) logger.debug('XML is valid') return (True, xml_doc, '') except etree.DocumentInvalid as e: logger.exception(e) logger.error('Fail to parse XML') return (False, xml_doc, str(e))
def upload(self, fp, size=None, progress_callback=None): """ Upload file contents. fp can be any file-like object, but if you don't specify it's size in advance it must support tell and seek methods. Progress callback is optional - if provided, it should match signature of ProgressCallbacks.upload_progress """ if isinstance(fp, bytes): fp = BytesIO(fp) elif isinstance(fp, str): fp = BytesIO(fp.encode('utf-8')) if size is None: size = base.get_file_size(fp) if size < self._upload_chunk_size: # simple, one request upload retries = max(self._upload_retries, 1) while retries > 0: url = self._client.get_url(self._url_template_content, path=self.path) chunk = base._FileChunk(fp, 0, size) r = self._client.POST(url, data=chunk, headers={'Content-length': str(size)}) exc.default.check_response(r) server_sha = r.headers['X-Sha512-Checksum'] our_sha = chunk.sha.hexdigest() if server_sha == our_sha: break retries -= 1 # TODO: retry network errors too if retries == 0: raise exc.ChecksumError("Failed to upload file", {}) else: # chunked upload return self._chunked_upload(fp, size, progress_callback)
def _add_file(self, name, size, data): if isinstance(data, bytes): data = BytesIO(data) elif isinstance(data, str): data = BytesIO(data.encode()) info = tarfile.TarInfo(name) info.size = size self.archive.addfile(info, data)
def __init__(self, image, symbols): image = BytesIO(b64decode(image.encode('ascii'))) super(CaissedepargneKeyboard, self).__init__(symbols, 5, 3, image, self.color, convert='RGB')
def sink(self, path: Optional[str] = None, name: Optional[str] = None, file: Optional[Union[TextIO, BytesIO]] = None, content: Optional[Union[str, bytes]] = None): file_content = content or self.scrape_result['content'] file_path = None file_extension = None if name: name_parts = name.split('.') if len(name_parts) > 1: file_extension = name_parts[-1] if not file: if file_extension is None: try: mime_type = self.scrape_result['response_headers'][ 'content-type'] except KeyError: mime_type = 'application/octet-stream' if ';' in mime_type: mime_type = mime_type.split(';')[0] file_extension = '.' + mime_type.split('/')[1] if not name: name = self.config['url'].split('/')[-1] if name.find(file_extension) == -1: name += file_extension file_path = path + '/' + name if path is not None else name if file_path == file_extension: url = re.sub(r'(https|http)?://', '', self.config['url']).replace('/', '-') if url[-1] == '-': url = url[:-1] url += file_extension file_path = url file = open(file_path, 'wb') if isinstance(file_content, str): file_content = BytesIO(file_content.encode('utf-8')) elif isinstance(file_content, bytes): file_content = BytesIO(file_content) file_content.seek(0) with file as f: shutil.copyfileobj(file_content, f, length=131072) logger.info('file %s created' % file_path)
def request_content(): content = request_key('content') if content: content = BytesIO(content.encode('utf-8')) return content, request_key('filename') fs = request.files.get('content', request.files.get('c')) if fs: return fs.stream, request_key('filename') or fs.filename
def request_content(): content = request_key('content') or request_key('file') if content: content = BytesIO(content.encode('utf-8')) return content, request_key('filename') fs = request_files('content') or request_files('file') if fs: return fs.stream, request_key('filename') or fs.filename return None, None
def request_content(): content = request_key('content') if content: content = BytesIO(content.encode('utf-8')) return content, request_key('filename') fs = request.files.get('content', request.files.get('c')) if fs: return fs.stream, request_key('filename') or fs.filename return None, None
def _set_data(self, data): if isinstance(data, str): data = BytesIO(data.encode('utf-8')) elif isinstance(data, bytes): data = BytesIO(data) File._set_data(self, data) if hasattr(data, 'seek'): data.seek(0) img = Image.open(data) self.image_size = img.size
def parse_tsv(text): """ Python 2 and 3 compatible TSV parser that returns a list of dictionary objects. """ if six.PY2: text = BytesIO(text.encode('utf-8')) else: text = text.splitlines() return list(csv.DictReader(text, dialect='excel-tab'))
def __init__(self, data): """Parse the given data string or file-like as a metadata block. The metadata header should not be included.""" if data is not None: if isinstance(data, text_type): data = BytesIO(data.encode('utf-8')) elif isinstance(data, byte_types): data = BytesIO(data) elif not hasattr(data, 'read'): raise TypeError( "StreamInfo requires string data or a file-like") self.load(data)
def put_attachment(self, content, name=None, content_type=None, content_length=None, domain=None, type_code=None): """Put attachment in blob database See `get_short_identifier()` for restrictions on the upper bound for number of attachments per object. :param content: String or file object. """ db = get_blob_db() if name is None: name = getattr(content, "name", None) if name is None: raise InvalidAttachment("cannot save attachment without name") if self._id is None: raise ResourceNotFound("cannot put attachment on unidentified document") if hasattr(self, "domain"): if domain is not None and self.domain != domain: raise ValueError("domain mismatch: %s != %s" % (self.domain, domain)) domain = self.domain elif domain is None: raise ValueError("domain attribute or argument is required") old_meta = self.blobs.get(name) if isinstance(content, six.text_type): content = BytesIO(content.encode("utf-8")) elif isinstance(content, bytes): content = BytesIO(content) # do we need to worry about BlobDB reading beyond content_length? meta = db.put( content, domain=domain or self.domain, parent_id=self._id, name=name, type_code=(self._blobdb_type_code if type_code is None else type_code), content_type=content_type, ) self.external_blobs[name] = BlobMetaRef( key=meta.key, blobmeta_id=meta.id, content_type=content_type, content_length=meta.content_length, ) if self._migrating_blobs_from_couch and self._attachments: self._attachments.pop(name, None) if self._atomic_blobs is None: self.save() if old_meta and old_meta.key: db.delete(key=old_meta.key) elif old_meta and old_meta.key: self._atomic_blobs[name].append(old_meta.key) return True
def put_attachment(self, content, name=None, content_type=None, content_length=None, domain=None, type_code=None): """Put attachment in blob database See `get_short_identifier()` for restrictions on the upper bound for number of attachments per object. :param content: String or file object. """ db = get_blob_db() if name is None: name = getattr(content, "name", None) if name is None: raise InvalidAttachment("cannot save attachment without name") if self._id is None: raise ResourceNotFound("cannot put attachment on unidentified document") if hasattr(self, "domain"): if domain is not None and self.domain != domain: raise ValueError("domain mismatch: %s != %s" % (self.domain, domain)) domain = self.domain elif domain is None: raise ValueError("domain attribute or argument is required") old_meta = self.blobs.get(name) if isinstance(content, str): content = BytesIO(content.encode("utf-8")) elif isinstance(content, bytes): content = BytesIO(content) # do we need to worry about BlobDB reading beyond content_length? meta = db.put( content, domain=domain or self.domain, parent_id=self._id, name=name, type_code=(self._blobdb_type_code if type_code is None else type_code), content_type=content_type, ) self.external_blobs[name] = BlobMetaRef( key=meta.key, blobmeta_id=meta.id, content_type=content_type, content_length=meta.content_length, ) if self._migrating_blobs_from_couch and self._attachments: self._attachments.pop(name, None) if self._atomic_blobs is None: self.save() if old_meta and old_meta.key: db.delete(key=old_meta.key) elif old_meta and old_meta.key: self._atomic_blobs[name].append(old_meta.key) return True
def encode_raw(self, data): """ encode complex data for raw output """ def _rewind(fid): if hasattr(fid, 'seek'): fid.seek(0) return data mime_type = getattr(data, 'mime_type', None) encoding = getattr(data, 'encoding', None) schema = getattr(data, 'schema', None) format_ = self.get_format(mime_type, encoding, schema) text_encoding = getattr(format_, 'text_encoding', 'utf-8') if format_ is None: raise ValueError("Invalid format specification! mime_type=%r, " "encoding=%r, schema=%r" % (mime_type, encoding, schema)) if isinstance(data, CDObject): data = data.data if format_.is_xml: data = BytesIO( etree.tostring(data, pretty_print=False, xml_declaration=True, encoding=text_encoding)) content_type = "%s; charset=%s" % (format_.mime_type, text_encoding) elif format_.is_json: data = BytesIO( json.dumps(data, ensure_ascii=False).encode(text_encoding)) content_type = "%s; charset=%s" % (format_.mime_type, text_encoding) elif format_.is_text: if isinstance(data, (CDTextBuffer, CDAsciiTextBuffer)): data.text_encoding = text_encoding else: source_text_encoding = getattr(data, 'text_encoding', None) if source_text_encoding != text_encoding: data = _rewind(data).read() if source_text_encoding is not None: data = data.decode(source_text_encoding) data = BytesIO(data.encode(text_encoding)) content_type = "%s; charset=%s" % (format_.mime_type, text_encoding) else: # generic binary byte-stream if format_.encoding is not None: data_out = BytesIO() for chunk in format_.encode(_rewind(data)): # if isinstance(chunk, binary_type): # chunk = str(chunk,'utf-8') data_out.write(chunk) data = data_out content_type = format_.mime_type return _rewind(data), content_type
def __init__(self, data=None, *args, **kwargs): # Collect the args to pass to load, this lets child classes # override just load and get equivalent magic for the # constructor. if data is not None: if isinstance(data, text_type): data = BytesIO(data.encode('utf-8')) elif isinstance(data, byte_types): data = BytesIO(data) elif not hasattr(data, 'read'): raise TypeError("VComment requires string data or a file-like") self.load(data, *args, **kwargs)
def _decode_obj(cls, obj_str): try: if obj_str is None: return None if isinstance(obj_str, str): obj_str = BytesIO(obj_str.encode('utf-8')) result = JSONParser().parse(obj_str) return result except Exception as e: raise NotImplementedError("only can decode json strings, received " "type: {}, repr: {}".format( type(obj_str), repr(obj_str))) from e
def test_sms_resource(self, request_mock): """Test the SMS resource PUT method, using the fake response returned by util.send_sms().""" sms = { "message": self.message['message'], "sms": self.subscriber['sms'] } #Create the mock response. dummyResponse = { "message-count": "1", "messages": [{ "message-id": "TEST-MESSAGE-ID", "message-price": "0.03330000", "network": "0000", "remaining-balance": "3.58010000", "status": "0", "to": sms['sms'] }] } dummyResponse = json.dumps(dummyResponse) dummyResponse = BytesIO( dummyResponse.encode() ) request_mock.return_value = dummyResponse #Test PUT method. put_response = self.app.put( '/sms', data=sms ) put_response = json.loads( put_response.data.decode('UTF-8') ) params = { 'api_key': meerkat_hermes.app.config['NEXMO_PUBLIC_KEY'], 'api_secret': meerkat_hermes.app.config['NEXMO_PRIVATE_KEY'], 'to': sms['sms'], 'from': meerkat_hermes.app.config['FROM'], 'text': sms['message'] } self.assertTrue( request_mock.called ) request_mock.assert_called_with( 'https://rest.nexmo.com/sms/json?' + urllib.parse.urlencode(params) ) self.assertEquals( put_response['messages'][0]['status'], '0' ) self.assertEquals( put_response['messages'][0]['to'], sms['sms'] ) #Check that the message has been logged properly. log_response = self.log.get_item( Key={ 'id': put_response['log_id'] } ) self.assertEquals( log_response['Item']['destination'][0], sms['sms'] ) #Delete the message from the log delete_response = self.app.delete( '/log/'+put_response['log_id'] )
def test_vrt_vsifile(data_dir, path_white_gemini_iv_vrt): """Successfully read an in-memory VRT""" with open(path_white_gemini_iv_vrt) as vrtfile: source = vrtfile.read() source = source.replace('<SourceFilename relativeToVRT="1">389225main_sw_1965_1024.jpg</SourceFilename>', '<SourceFilename relativeToVRT="0">{}/389225main_sw_1965_1024.jpg</SourceFilename>'.format(data_dir)) source = BytesIO(source.encode('utf-8')) with FilePath(source) as vsifile: with vsifile.open() as src: assert src.driver == 'VRT' assert src.count == 3 assert src.dtypes == ('uint8', 'uint8', 'uint8') assert src.read().shape == (3, 768, 1024)
def upload_to_mapbox(username, datasetname, geojson): if type(geojson) == dict: geojson = json.dumps(geojson) if type(geojson) == str: geojson = BytesIO(geojson.encode()) # see: https://www.mapbox.com/api-documentation/#uploads # Acquisition of credentials, staging of data, and upload # finalization is done by a single method in the Python SDK. service = Uploader() dataset_name = '{}.{}'.format(username, datasetname) resp = service.upload(geojson, dataset_name) return resp
def load_string(xml_str, validate=True): """Generate a python object :param xml_str: the XML file as string :type xml_str: str :param validate: validate the XML before generating the python object. :type validate: bool :return: the generated python object :rtype: :class:`Element` """ if not isinstance(xml_str, BytesIO): # TODO: Get encoding from the dtd file (xml tag). xml_str = BytesIO(xml_str.encode('utf-8')) return load(xml_str, validate)
def testParseHeader(self): # Test parse_header() for version 2 header = str(self.headerClass(self.defaultVersion)) ofx = header + self.body ofx = BytesIO(ofx.encode("utf8")) ofxheader, body = ofxtools.header.parse_header(ofx) self.assertEqual(ofxheader.ofxheader, 200) self.assertEqual(ofxheader.version, self.defaultVersion) self.assertEqual(ofxheader.security, "NONE") self.assertEqual(ofxheader.oldfileuid, "NONE") self.assertEqual(ofxheader.newfileuid, "NONE") self.assertEqual(body, self.body)
def build_tree(xml_string): """ Returns a lxml etree from an XML string (xml, xsd...) Args: xml_string: Returns: """ try: xml_string = BytesIO(xml_string.encode('utf-8')) except Exception: xml_string = BytesIO(xml_string) return etree.parse(xml_string)
def build(self): from docker import Client from io import BytesIO dockerhost = os.environ.get("DOCKER_HOST", "tcp://127.0.0.1:2375") dockerfile = super(DockerBuild, self).build() dockerfile = BytesIO(dockerfile.encode('utf-8')) cli = Client(dockerhost) builder = cli.build(fileobj=dockerfile, rm=True, tag="bones") for line in builder: line = json.loads(line) if len(line.keys()) == 1 and "stream" in line: sys.stdout.write(line["stream"]) sys.stdout.flush() else: print(line)
def upload_file(self, filename, src, mkdir=False): if isinstance(src, str): src = BytesIO(src.encode("utf-8")) if mkdir: directory = posixpath.dirname(filename) if directory: self.mkdir(directory, recursive=True) if not isinstance(filename, str): filename = filename.decode("utf-8") try: self.con.storbinary("STOR " + filename, src, blocksize=32768) except FTPError as e: self.log_buffer.append(str(e)) return False return True
def testNoLineBreaksAnywhere(self): # Some FIs apparently return OFX data with no line breaks anywhere, not # even in the OFX header. We're cool with that. # # Issue #89 header = ("OFXHEADER:100" "DATA:OFXSGML" "VERSION:102" "SECURITY:NONE" "ENCODING:USASCII" "CHARSET:1252" "COMPRESSION:NONE" "OLDFILEUID:NONE" "NEWFILEUID:NONE") body = ( "<OFX><SIGNONMSGSRSV1><SONRS>" "<STATUS><CODE>0<SEVERITY>INFO<MESSAGE>SUCCESS</STATUS>" "<DTSERVER>20200818065106.132[-7:PDT]<LANGUAGE>ENG<FI><ORG>WF<FID>3000</FI>" "<SESSCOOKIE>a6a12496-682d-42b2-aafa-ed973c406a17-08182020075105922" "<INTU.BID>3000<INTU.USERID>jdoe</SONRS></SIGNONMSGSRSV1>" "<BANKMSGSRSV1><STMTTRNRS><TRNUID>0" "<STATUS><CODE>0<SEVERITY>INFO<MESSAGE>SUCCESS</STATUS>" "<STMTRS><CURDEF>USD<BANKACCTFROM><BANKID>121042882<ACCTID>5555555555" "<ACCTTYPE>CHECKING</BANKACCTFROM>" "<BANKTRANLIST><DTSTART>20200101120000.000[-8:PST]" "<DTEND>20200810110000.000[-7:PDT]" "<STMTTRN><TRNTYPE>CREDIT<DTPOSTED>20200403110000.000[-7:PDT]" "<TRNAMT>507500.00<FITID>202004031<NAME>TD AMERITRADE CLEARING" "<MEMO>WT SEQ555555 /ORG=JOHN DOE SRF# EC55555555555 TRN#55555555555 RFB# 55555555555" "</STMTTRN></BANKTRANLIST>" "<LEDGERBAL><BALAMT>56498.04<DTASOF>20200817110000.000[-7:PDT]</LEDGERBAL>" "<AVAILBAL><BALAMT>56498.04<DTASOF>20200817110000.000[-7:PDT]</AVAILBAL>" "</STMTRS></STMTTRNRS></BANKMSGSRSV1></OFX>") ofx = header + body ofx = BytesIO(ofx.encode("utf8")) ofxheader, body_ = ofxtools.header.parse_header(ofx) self.assertEqual(ofxheader.ofxheader, 100) self.assertEqual(ofxheader.data, "OFXSGML") self.assertEqual(ofxheader.version, 102) self.assertEqual(ofxheader.security, "NONE") self.assertEqual(ofxheader.encoding, "USASCII") self.assertEqual(ofxheader.charset, "1252") self.assertEqual(ofxheader.compression, "NONE") self.assertEqual(ofxheader.oldfileuid, "NONE") self.assertEqual(ofxheader.newfileuid, "NONE") self.assertEqual(body, body_)
def roundtrip(f): """ Test roundtrip for `untokenize`. `f` is an open file or a string. The source code in f is tokenized, converted back to source code via tokenize.untokenize(), and tokenized again from the latter. The test fails if the second tokenization doesn't match the first. """ if isinstance(f, str): f = BytesIO(f.encode('utf-8')) token_list = list(tokenize(f.readline)) f.close() tokens1 = [tok[:2] for tok in token_list] new_bytes = untokenize(tokens1) readline = (line for line in new_bytes.splitlines(1)).__next__ tokens2 = [tok[:2] for tok in tokenize(readline)] return tokens1 == tokens2
def build_tree(xml_string, parser=None): """Returns a lxml etree from an XML string (xml, xsd...) Args: xml_string: parser: Returns: """ try: xml_string = BytesIO(xml_string.encode("utf-8")) except Exception: xml_string = BytesIO(xml_string) if not parser: parser = XMLParser(remove_blank_text=True) return etree.parse(xml_string, parser=parser)
def testParseHeader(self): # Test parse_header() for version 1 header = str(self.headerClass(self.defaultVersion)) ofx = header + self.body ofx = BytesIO(ofx.encode("utf8")) ofxheader, body = ofxtools.header.parse_header(ofx) self.assertEqual(ofxheader.ofxheader, 100) self.assertEqual(ofxheader.data, "OFXSGML") self.assertEqual(ofxheader.version, self.defaultVersion) self.assertEqual(ofxheader.security, "NONE") self.assertEqual(ofxheader.encoding, "USASCII") self.assertEqual(ofxheader.charset, "NONE") self.assertEqual(ofxheader.compression, "NONE") self.assertEqual(ofxheader.oldfileuid, "NONE") self.assertEqual(ofxheader.newfileuid, "NONE") self.assertEqual(body, self.body)
def testParseHeaderSingleQuotedDeclarationData(self): # The XML spec allows data to be quoted within either single or double quotes # Make sure that single-quoted data in the XML declaration is captured by # ``ofxtools.header.XML_REGEX`` header = str(self.headerClass(self.defaultVersion)) header = ("<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + header[header.find("<?OFX"):]) ofx = header + self.body ofx = BytesIO(ofx.encode("utf8")) ofxheader, body = ofxtools.header.parse_header(ofx) self.assertEqual(ofxheader.ofxheader, 200) self.assertEqual(ofxheader.version, self.defaultVersion) self.assertEqual(ofxheader.security, "NONE") self.assertEqual(ofxheader.oldfileuid, "NONE") self.assertEqual(ofxheader.newfileuid, "NONE") self.assertEqual(body, self.body)
def content_file(content_id, filename): """ Serve file from zipball with specified id """ content_dir = request.app.config['content.contentdir'] zippath = downloads.get_zip_path(content_id, content_dir) try: metadata, content = downloads.get_file(zippath, filename, no_read=True) except downloads.ContentError as err: logging.error(err) abort(404) size = metadata.file_size timestamp = os.stat(zippath)[stat.ST_MTIME] archive = open_archive() if filename.endswith('.html') and archive.needs_formatting(content_id): logging.debug("Patching HTML file '%s' with Librarian stylesheet" % ( filename)) size, content = patch_content.patch(content.read()) content = StringIO(content.encode('utf8')) return send_file(content, filename, size, timestamp)
async def backups(self): name = datetime.now().__format__("%d%m%y-%H:%M") SHELL = os.getenv("SHELL") or "/bin/bash" sequence = [SHELL, '-c', """pg_dump -U dredd -h localhost "dredd v3" > "backups/{0}.sql" """.format(name)] subprocess.Popen(sequence, stdout=subprocess.PIPE, stderr=subprocess.PIPE) content = 'Backup created on {0}'.format(name) receiver = self.bot.config.BACKUP_RECEIVER ch = self.bot.get_channel(679647378210291832) await asyncio.sleep(5) with open(f'backups/{name}.sql', 'r', encoding='utf8') as f: backup = f.read() backup = BytesIO(backup.encode('utf8')) if not backup: return await ch.send(f"{self.bot.get_user(345457928972533773).mention} Backup `{name}.sql` is empty!", allowed_mentions=discord.AllowedMentions(users=True)) else: await self.client.send(receiver, content, subject="Database Backup", bcc=None, attachment_bytes=backup.read(), attachment_name=f"{name}.sql") return await ch.send(f"Created backup `{name}.sql`")
def request_content(): content_type = http.parse_options_header(request.headers.get("Content-Type", "")) if content_type: content_type, _ = content_type if content_type == "application/json": content = request.json.get("c") if content: content = BytesIO(content.encode("utf-8")) return content, request.json.get("filename") c = request.form.get("c") if c: return BytesIO(c.encode("utf-8")), None fs = request.files.get("c") if fs: return fs.stream, fs.filename return None, None
def request_content(): content_type = http.parse_options_header(request.headers.get('Content-Type', '')) if content_type: content_type, _ = content_type if content_type == 'application/json': content = request.json.get('c') if content: content = BytesIO(content.encode('utf-8')) return content, request.json.get('filename') c = request.form.get('c') if c: return BytesIO(c.encode('utf-8')), None fs = request.files.get('c') if fs: return fs.stream, fs.filename return None, None
def put(self, path, data, metadata=None): """Store a File object, stream, unicode string, or byte string. Optionally attach metadata to it. """ # Coerce strings to in-memory Byte streams. if isinstance(data, string_types): data = BytesIO(data.encode()) # Coerce python 3 explicit byte strings if isinstance(data, bytes): data = BytesIO(data) k = self.bucket.new_key(path) if metadata: k.update_metadata(metadata) k.set_contents_from_file( data, reduced_redundancy=self.reduced_redundancy, encrypt_key=self.encrypted)
def testParseHeaderUnicode(self): """ Test parse_header() with UTF-8 charset """ # Issue #49 header = str( self.headerClass(self.defaultVersion, encoding="UTF-8", charset="NONE") ) ofx = header + self.body_utf8 ofx = BytesIO(ofx.encode("utf_8")) ofxheader, body = ofxtools.header.parse_header(ofx) self.assertEqual(ofxheader.ofxheader, 100) self.assertEqual(ofxheader.data, "OFXSGML") self.assertEqual(ofxheader.version, self.defaultVersion) self.assertEqual(ofxheader.security, "NONE") self.assertEqual(ofxheader.encoding, "UTF-8") self.assertEqual(ofxheader.charset, "NONE") self.assertEqual(ofxheader.compression, "NONE") self.assertEqual(ofxheader.oldfileuid, "NONE") self.assertEqual(ofxheader.newfileuid, "NONE") self.assertEqual(body, self.body_utf8)
def testParseHeaderLatin1(self): """ Test parse_header() with ISO-8859-1 charset """ header = str( self.headerClass( self.defaultVersion, encoding="UTF-8", charset="ISO-8859-1" ) ) ofx = header + self.body ofx = BytesIO(ofx.encode("latin_1")) ofxheader, body = ofxtools.header.parse_header(ofx) self.assertEqual(ofxheader.ofxheader, 100) self.assertEqual(ofxheader.data, "OFXSGML") self.assertEqual(ofxheader.version, self.defaultVersion) self.assertEqual(ofxheader.security, "NONE") self.assertEqual(ofxheader.encoding, "UTF-8") self.assertEqual(ofxheader.charset, "ISO-8859-1") self.assertEqual(ofxheader.compression, "NONE") self.assertEqual(ofxheader.oldfileuid, "NONE") self.assertEqual(ofxheader.newfileuid, "NONE") self.assertEqual(body, self.body)
def testParseHeaderV2NoNewlineBetweenHeaderAndBody(self): """ OFXv2 doesn't need newline between header and message body """ # Issue #47 header = str(self.headerClass(self.defaultVersion)) ofx = header.strip() + self.body ofx = BytesIO(ofx.encode("ascii")) ofxheader, body = ofxtools.header.parse_header(ofx) self.assertEqual(ofxheader.ofxheader, 200) self.assertEqual(ofxheader.version, self.defaultVersion) self.assertEqual(ofxheader.security, "NONE") self.assertEqual(ofxheader.oldfileuid, "NONE") self.assertEqual(ofxheader.newfileuid, "NONE") self.assertEqual(body, self.body) # Make sure body can actually be parsed builder = ofxtools.Parser.TreeBuilder() builder.feed(body) root = builder.close() self.assertIsInstance(root, Element) self.assertEqual(root.tag, "OFX") self.assertIsNone(root.text) self.assertEqual(len(root), 1)
def set(self, content): if isinstance(content, Body): self.mimetype = content.mimetype self.data = content.data self.chunked = content.chunked self.trailer = content.trailer self.content = content.content return if not content: content = BytesIO() elif isinstance(content, (BytesIO, file)): if content.closed: raise TypeError('I/O operation on closed file.') elif isinstance(content, Unicode): content = BytesIO(content.encode(self.encoding)) elif isinstance(content, bytes): content = BytesIO(content) elif isinstance(content, bytearray): content = BytesIO(bytes(content)) elif not hasattr(content, '__iter__'): raise TypeError('Content must be iterable.') self.content = content
def test_publish_resource(self, mock_sms_response): """Test the Publish resource PUT method.""" def clones(object, times=None): #A generator to yield clones of an object, infitnely or upto n times. #Used to generate new nexmo response objects for the mock_sms_response if times is None: while True: yield copy.copy(object) else: for i in range(times): yield copy.copy(object) #Createfour test subscribers, each with subscriptions to a different list of topics. topic_lists = [['Test1', 'Test2'], ['Test1'], ['Test2'], ['Test3']] subscriber_ids = [] for topics in topic_lists: #Create a variation on the test subscriber subscriber = self.subscriber.copy() subscriber['topics'] = topics subscriber['verified'] = True subscriber['first_name'] += str(topic_lists.index(topics)) #Remove the SMS field from two of the subscribers if( topic_lists.index(topics) % 2 != 0 ): del subscriber['sms'] #Add the subscriber to the database. subscribe_response = self.app.put('/subscribe', data=subscriber ) subscriber_ids.append(json.loads( subscribe_response.data.decode('UTF-8') )['subscriber_id']) #Create the mock SMS response. dummyResponse = { "message-count": "1", "messages": [{ "message-id": "TEST-MESSAGE-ID", "message-price": "0.03330000", "network": "0000", "remaining-balance": "3.58010000", "status": "0", "to": self.subscriber['sms'] }] } dummyResponse = json.dumps(dummyResponse) dummyResponse = BytesIO( dummyResponse.encode() ) mock_sms_response.side_effect = clones(dummyResponse) #Create the message. message = self.message.copy() message['html-message'] = message.pop('html') message['medium'] = ['email','sms'] #Keep track of the message IDs so we can delete the logs afterwards. message_ids = [] #Test the PUT Method with different calls. #----------------------------------------- #Publish the test message to topic Test4. message['topics'] = ['Test4'] message['id'] = "testID1" message_ids.append( message['id'] ) put_response = self.app.put( '/publish', data=message ) put_response = json.loads( put_response.data.decode('UTF-8') ) print( put_response ) #No subscribers have subscribed to 'Test4'. #No messages should be sent #Check that no messages have been sent and that the sms response has not been called. self.assertEquals( len(put_response), 0 ) self.assertFalse( mock_sms_response.called ) #Add a subscription without a subscriber, to check that the faulty data is properly handled. self.subscriptions.put_item( Item={ "subscriberID": "TESTID", "subscriptionID": "TESTSUBSCRIPTION", "topicID":"Test4" } ) message['id'] = "testID1b" message_ids.append( message['id'] ) put_response = self.app.put( '/publish', data=message ) put_response = json.loads( put_response.data.decode('UTF-8') ) print( "Put response: " + str(put_response) ) #One response informing us that the subscriber TESTID is deleted. self.assertEquals( len(put_response), 1 ) self.assertFalse( mock_sms_response.called ) #Publish the test message to topic Test3. message['topics'] = ['Test3'] message['id'] = "testID2" message_ids.append( message['id'] ) put_response = self.app.put( '/publish', data=message ) put_response = json.loads( put_response.data.decode('UTF-8') ) print( "Response to publishing message to topic: " + str(message['topics']) + "\n" + str(put_response) ) #Only subscriber 4 has subscription to 'Test3'. #Subscriber 4 hasn't given an SMS number, so only one email is sent. #Check only one email is sent and no sms calls are made. self.assertEquals( len(put_response), 1 ) self.assertFalse( mock_sms_response.called ) self.assertEquals( put_response[0]['Destination'][0], self.subscriber['email'] ) #Publish the test message to topic Test1. message['topics'] = ['Test1'] message['id'] = "testID3" message_ids.append( message['id'] ) put_response = self.app.put( '/publish', data=message ) put_response = json.loads( put_response.data.decode('UTF-8') ) print( "Response to publishing message to topic: " + str(message['topics']) + "\n" + str(put_response) ) #Subscriber 1 and 2 have subscriptions to 'Test1'. #Subscriber 2 hasn't given an SMS number, so two emails and one sms are sent. #Check three messages sent in total and sms mock called once. self.assertEquals( len(put_response), 3 ) self.assertTrue( mock_sms_response.call_count == 1 ) #Publish the test message to both topics Test1 and Test2. message['topics'] = ['Test1', 'Test2'] message['id'] = "testID4" message_ids.append( message['id'] ) put_response = self.app.put( '/publish', data=message ) put_response = json.loads( put_response.data.decode('UTF-8') ) print( "Response to publishing message to topic: " + str(message['topics']) + "\n" + str(put_response) ) #Subscriber 1 has a subscription to both 'Test1' and 'Test2' so gets sms and email twice. #Subscriber 2 has a subscription to 'Test1' but doesn't have an sms, so gets just one email. #Subscriber 3 has a subscription to 'Test2' so gets one email and one sms message. #This results in 4 messages to subscriber 1, 1 to subscriber 2, and 2 to subscriber 3. #Check number of messages sent is 7 and that sms mock has been called ANOTHER 3 times. self.assertEquals( len(put_response), 7 ) self.assertTrue( mock_sms_response.call_count == 4 ) #Delete the logs. for message_id in message_ids: delete_response = self.app.delete( '/log/' + message_id ) #Delete the test subscribers. for subscriber_id in subscriber_ids: delete_response = self.app.delete( '/subscribe/' + subscriber_id )