def parse_stream(self, data): hdr = header.parse_stream(data) if hdr.signature[0] == "C": import zlib data.seek(0) data = StringIO(data.read(8) + zlib.decompress(data.read())) return Struct.parse_stream(self, data)
def add_packet(self, new_packet): """ Adds the packet to the unfinished object. Updates all things appropriately :param string newPacket: The incoming packet """ packet = unpack(new_packet) reader = StringIO(packet[1]) # Get the actual packet data as a StringIO packet_number = _bin_unpack(reader.read(4)) # Read the 4-byte number of the packet # Try to write the info to the correct place self.info_buffer.seek(_UnfinishedPacket.MAX_PACKET_SIZE * packet_number) self.info_buffer.write(reader.read()) reader.close() self.info_buffer.seek(0) # Update incoming packet tracking data if(packet_number > self.last_received): self.last_received = packet_number for num in range(self.last_received,packet_number): self.missing_packets.append(num) else: self.missing_packets.remove(packet_number) # See if all packets are received. if(self.last_received == self.num_packets-1 and not self.missing_packets): self.done = True
class TBufferedTransport(TTransportBase,CReadableTransport): """Class that wraps another transport and buffers its I/O. The implementation uses a (configurable) fixed-size read buffer but buffers all writes until a flush is performed. """ DEFAULT_BUFFER = 4096 def __init__(self, trans, rbuf_size = DEFAULT_BUFFER): self.__trans = trans self.__wbuf = StringIO() self.__rbuf = StringIO("") self.__rbuf_size = rbuf_size def isOpen(self): return self.__trans.isOpen() def open(self): return self.__trans.open() def close(self): return self.__trans.close() def read(self, sz): ret = self.__rbuf.read(sz) if len(ret) != 0: return ret self.__rbuf = StringIO(self.__trans.read(max(sz, self.__rbuf_size))) return self.__rbuf.read(sz) def write(self, buf): self.__wbuf.write(buf) def flush(self): out = self.__wbuf.getvalue() # reset wbuf before write/flush to preserve state on underlying failure self.__wbuf = StringIO() self.__trans.write(out) self.__trans.flush() # Implement the CReadableTransport interface. @property def cstringio_buf(self): return self.__rbuf def cstringio_refill(self, partialread, reqlen): retstring = partialread if reqlen < self.__rbuf_size: # try to make a read of as much as we can. retstring += self.__trans.read(self.__rbuf_size) # but make sure we do read reqlen bytes. if len(retstring) < reqlen: retstring += self.__trans.readAll(reqlen - len(retstring)) self.__rbuf = StringIO(retstring) return self.__rbuf
def cacheImageFile(filename): "Processes the image as if for encoding, saves to a file ending in AHX" from PIL import Image import zlib img1 = Image.open(filename) img = img1.convert('RGB') imgwidth, imgheight = img.size code = [] code.append('BI') # begin image # this describes what is in the image itself code.append('/W %s /H %s /BPC 8 /CS /RGB /F [/A85 /Fl]' % (imgwidth, imgheight)) code.append('ID') #use a flate filter and Ascii Base 85 raw = getattr(img, 'tobytes', img.tostring)() assert len(raw) == imgwidth * imgheight, "Wrong amount of data for image" compressed = zlib.compress(raw) #this bit is very fast... encoded = _AsciiBase85Encode(compressed) #...sadly this isn't #write in blocks of 60 characters per line outstream = StringIO(encoded) dataline = outstream.read(60) while dataline != "": code.append(dataline) dataline = outstream.read(60) code.append('EI') #save it to a file cachedname = os.path.splitext(filename)[0] + '.a85' f = open(cachedname, 'wb') f.write(string.join(code, LINEEND) + LINEEND) f.close() print('cached image as %s' % cachedname)
def run_test(conf_url=querymod.configuration.get('url', None)): """ Run the unit tests and make sure the server is online.""" if conf_url is None: raise ValueError("Please create a local api_config.json file in the " "root directory of the repository with 'url' defined " "with the root URL of the server. (No /rest or " "/jsonrpc should be present.) Or provide URL on " "command line.") # Tell the test framework where to query global url url = conf_url results = StringIO() # Run the test demo_test = unittest.TestLoader().loadTestsFromTestCase(TestAPI) unittest.TextTestRunner(stream=results).run(demo_test) # See if the end of the results says it passed results.seek(results.tell() - 3) if results.read() == "OK\n": sys.exit(0) else: results.seek(0) print(results.read()) sys.exit(1)
def filename_to_docname(filename): """ Convert the filename of a saved document to a documentname. Filenames have the form "cnl-[documentname].save" where [documentname] is the name of the document with escaped special characters Positional arguments: filename -- Name of the file with escaped special characters Return value: Name of the document without escaped special characters. """ result = "" input = StringIO(filename[4:-5]) while True: char = input.read(1) if char == "": break elif char == ':': charcode = "" while len(charcode) == 0 or charcode[-1] != ';': charcode += input.read(1) char = chr(int(charcode[:-1], 16)) result += char return result
def _open(self): header = StringIO(self.fp.read(24)) magic = header.read(4) if magic != "FTEX": raise ValueError("not a FTEX file") version = unpack("i", header.read(4)) self.size = unpack("ii", header.read(8)) linesize = (self.size[0] + 3) / 4 * 8 mipmap_count, format_count = unpack("ii", header.read(8)) self.mode = "RGB" self.tile = [] for i in range(format_count): format, where = unpack("ii", self.fp.read(8)) if format == 0: data = [] self.fp.seek(where) size, = unpack("i", self.fp.read(4)) for yb in xrange((self.size[1] + 3) / 4): decoded = dxtc.decodeDXT1(self.fp.read(linesize)) for d in decoded: data.append(d) data = "".join(data[:self.size[1]]) self.im = Image.core.new(self.mode, self.size) return self.fromstring(data) elif format == 1: # Uncompressed RGB self.tile.append(("raw", (0, 0) + self.size, where+4, (self.mode, 0, 1))) else: raise ValueError("Invalid texture format (expected 0 or 1, got %i)" % (format))
class TFramedTransport(TTransportBase, CReadableTransport): """Class that wraps another transport and frames its I/O when writing.""" def __init__(self, trans,): self.__trans = trans self.__rbuf = StringIO() self.__wbuf = StringIO() def isOpen(self): return self.__trans.isOpen() def open(self): return self.__trans.open() def close(self): return self.__trans.close() def read(self, sz): ret = self.__rbuf.read(sz) if len(ret) != 0: return ret self.readFrame() return self.__rbuf.read(sz) def readFrame(self): buff = self.__trans.readAll(4) sz, = unpack('!i', buff) self.__rbuf = StringIO(self.__trans.readAll(sz)) def write(self, buf): self.__wbuf.write(buf) def flush(self): wout = self.__wbuf.getvalue() wsz = len(wout) # reset wbuf before write/flush to preserve state on underlying failure self.__wbuf = StringIO() # N.B.: Doing this string concatenation is WAY cheaper than making # two separate calls to the underlying socket object. Socket writes in # Python turn out to be REALLY expensive, but it seems to do a pretty # good job of managing string buffer operations without excessive copies buf = pack("!i", wsz) + wout self.__trans.write(buf) self.__trans.flush() # Implement the CReadableTransport interface. @property def cstringio_buf(self): return self.__rbuf def cstringio_refill(self, prefix, reqlen): # self.__rbuf will already be empty here because fastbinary doesn't # ask for a refill until the previous buffer is empty. Therefore, # we can start reading new frames immediately. while len(prefix) < reqlen: self.readFrame() prefix += self.__rbuf.getvalue() self.__rbuf = StringIO(prefix) return self.__rbuf
def test_magic_autopx_nonblocking(self): ip = get_ipython() v = self.client[-1] v.activate() v.block=False sio = StringIO() savestdout = sys.stdout sys.stdout = sio ip.magic_autopx() ip.run_cell('\n'.join(('a=5','b=10','c=0'))) ip.run_cell('print b') ip.run_cell("b/c") ip.run_code(compile('b*=2', '', 'single')) ip.magic_autopx() sys.stdout = savestdout sio.read() output = sio.buf.strip() self.assertTrue(output.startswith('%autopx enabled')) self.assertTrue(output.endswith('%autopx disabled')) self.assertFalse('ZeroDivisionError' in output) ar = v.get_result(-2) self.assertEquals(v['a'], 5) self.assertEquals(v['b'], 20) self.assertRaisesRemote(ZeroDivisionError, ar.get)
class FifoBuffer(io.TextIOBase): def __init__(self): self.buf = StringIO() self.len = 0 def read(self): """Reads data from buffer""" self.buf.seek(0) l = self.len self.len=0 res = self.buf.read(l) self.buf.seek(0) return res def write(self, arg): self.len = self.len + self.buf.write(arg) def peek(self): x = self.buf.tell() self.buf.seek(0) res = self.buf.read(x) self.buf.seek(x) return res
def normalize_cmd(): parser = ArgumentParser() parser.add_argument( '-C', '--check', dest='check', action='store_true', default=False, help=('Do not modify buildout file, ' 'only return if file would be modified')) parser.add_argument('configfiles', nargs='*', help=('The configfile to normalize in place, ' 'or "-" to read the config file from stdin ' 'and return the result to stdout')) args = parser.parse_args() if not len(args.configfiles): logger.exception('You must choose config files to normalize ', 'I won\'t guess them for you!') sys.exit(3) outstreams = {} for configfile in args.configfiles: if len(args.configfiles) == 1 and configfile == '-': instream = StringIO() instream.write(sys.stdin.read()) instream.seek(0) pipe = True else: instream = open(configfile, encoding='utf-8') pipe = False outstream = StringIO() outstreams[configfile] = outstream try: sort(instream, outstream) except Exception: logger.exception('Could not parse file %r', configfile) return sys.exit(3) else: instream.seek(0) outstream.seek(0) changed = instream.read() != outstream.read() outstream.seek(0) if not changed: if pipe: sys.stdout.write(outstream.read()) sys.exit(0) else: if args.check: logger.error('File is not normalized') sys.exit(3) else: if pipe: sys.stdout.write(outstream.read()) if not pipe: for outfile, outstream in outstreams.items(): open(outfile, 'w', encoding='utf-8').write(outstream.read())
def __decode_blp2(self): header = StringIO(self.fp.read(20 + 16 * 4 + 16 * 4)) magic, compression = unpack(b"<4si", header.read(8)) encoding, alphaDepth, alphaEncoding, hasMips = unpack(b"<4b", header.read(4)) self.size = unpack(b"<II", header.read(8)) offsets = unpack(b"<16I", header.read(16 * 4)) lengths = unpack(b"<16I", header.read(16 * 4)) palette_data = self.fp.read(256 * 4) self.mode = "RGB" self.tile = [] if compression == 1: # Uncompressed or DirectX compression data = [] self.fp.seek(offsets[0]) if encoding == 1: # uncompressed palette = getpalette(palette_data) _data = StringIO(self.fp.read(lengths[0])) while True: try: offset, = unpack(b"<B", _data.read(1)) except StructError: break b, g, r, a = palette[offset] data.append(pack(b"<BBB", r, g, b)) elif encoding == 2: # directx compression self.mode = "RGBA" if alphaDepth else "RGB" if alphaEncoding == 0: # DXT1 linesize = (self.size[0] + 3) / 4 * 8 for yb in xrange((self.size[1] + 3) / 4): decoded = dxtc.decodeDXT1(self.fp.read(linesize), alpha=alphaDepth) for d in decoded: data.append(d) elif alphaEncoding == 1: # DXT3 linesize = (self.size[0] + 3) / 4 * 16 for yb in xrange((self.size[1] + 3) / 4): decoded = dxtc.decodeDXT3(self.fp.read(linesize)) for d in decoded: data.append(d) elif alphaEncoding == 7: # DXT5 linesize = (self.size[0] + 3) / 4 * 16 for yb in xrange((self.size[1] + 3) / 4): decoded = dxtc.decodeDXT5(self.fp.read(linesize)) for d in decoded: data.append(d) else: raise ValueError("Expected alphaEncoding 0, 1 or 7, got %i instead" % (alphaEncoding)) self.im = Image.core.new(self.mode, self.size) data = "".join(data) self.fromstring(data)
def grade_string(expected, actual): """Grades a single response against the true (actual) output. Possible return values are: * MATCH_EXACT (perfect match) * MATCH_LINES (correct number of lines, non-formatting characters match) * MATCH_VALUES (non-formatting characters match) * MATCH_NONE (no match)""" # Convert to universal newlines, strip extraneous whitespace expected_io = StringIO(unicode(expected.strip()), newline=None) actual_io = StringIO(unicode(actual.strip()), newline=None) expected_str = expected_io.read() actual_str = actual_io.read() # Pefect match if expected_str == actual_str: return MATCH_EXACT format_chars = ['[', ']', ',', ' ', '\n', '"', '\''] table = dict.fromkeys(map(ord, format_chars), None) expected_io.seek(0) expected_lines = [line.strip() for line in expected_io.readlines()] actual_io.seek(0) actual_lines = [line.strip() for line in actual_io.readlines()] # Remove blank lines removed_blanks = False if len(expected_lines) != len(actual_lines): actual_lines = [line for line in actual_lines if len(line.strip()) > 0] removed_blanks = True # Check for line by line exact/partial match if len(expected_lines) == len(actual_lines): exact_match = True partial_match = False for (e_line, a_line) in zip(expected_lines, actual_lines): if e_line != a_line: exact_match = False if (e_line.translate(table).lower() == a_line.translate(table).lower()): partial_match = True else: partial_match = False break if exact_match: return MATCH_EXACT if not removed_blanks else MATCH_LINES elif partial_match: return MATCH_LINES # Check for partial match of values only if expected_str.translate(table).lower() == actual_str.translate(table).lower(): return MATCH_VALUES return MATCH_NONE
def xpm2png(value): xpm = '''/* XPM */ static char * ala_xpm[] = { "%s"};''' % '",\n"'.join(value) xpm = xpm.replace('None', '#ffffff').replace('Black', '#000000') io = StringIO(xpm) im = Image.open(io) outio = StringIO() im.save(outio, 'png') # I don't understand why, but this is needed to get data to buf outio.read() return outio.buf.encode('base64')
def tree_from_str(data): sio = StringIO(data) magic_str = sio.read(4) if magic_str != b'PFTR': raise RuntimeError("magic string not found") version_str = sio.read(4) version = struct.unpack("!I", version_str)[0] if version != TREE_VER_P1: raise RuntimeError("tree version not supported") return pickle.load(sio)
def toarray(string, wordsize=8): # Normalize first string = string.strip().replace(" ", "") pad_len = wordsize - (len(string) % wordsize) if pad_len < wordsize: string = ("0" * pad_len) + string stream = StringIO(string) chunks = [] chunk = stream.read(wordsize) while chunk: chunks.append(chunk) chunk = stream.read(wordsize) return ', '.join('0x{}'.format(x.lower()) for x in reversed(chunks))
def feed(self, markup): if isinstance(markup, str): markup = StringIO(markup) # Call feed() at least once, even if the markup is empty, # or the parser won't be initialized. data = markup.read(self.CHUNK_SIZE) self.parser.feed(data) while data != '': # Now call feed() on the rest of the data, chunk by chunk. data = markup.read(self.CHUNK_SIZE) if data != '': self.parser.feed(data) self.parser.close()
def grade_string(expected, actual): # Convert to universal newlines, strip extraneous whitespace expected_io = StringIO(unicode(expected.strip()), newline=None) actual_io = StringIO(unicode(actual.strip()), newline=None) expected_str = expected_io.read() actual_str = actual_io.read() # Pefect match if expected_str == actual_str: return "exact" table = dict.fromkeys(map(ord, FORMAT_CHARS), None) expected_io.seek(0) expected_lines = [line.strip() for line in expected_io.readlines()] actual_io.seek(0) actual_lines = [line.strip() for line in actual_io.readlines()] # Remove blank lines removed_blanks = False if len(expected_lines) != len(actual_lines): actual_lines = [line for line in actual_lines if len(line.strip()) > 0] removed_blanks = True # Check for line by line exact/partial match if len(expected_lines) == len(actual_lines): exact_match = True partial_match = False for (e_line, a_line) in zip(expected_lines, actual_lines): if e_line != a_line: exact_match = False if (e_line.translate(table).lower() == a_line.translate(table).lower()): partial_match = True else: partial_match = False break if exact_match: return "exact" if not removed_blanks else "line" elif partial_match: return "line" # Check for partial match of values only if expected_str.translate(table).lower() == actual_str.translate(table).lower(): return "values" return None
def RunLengthDecode(data): f = StringIO(data) decompressed = '' runLength = ord(f.read(1)) while runLength: if runLength < 128: decompressed += f.read(runLength + 1) if runLength > 128: decompressed += f.read(1) * (257 - runLength) if runLength == 128: break runLength = ord(f.read(1)) # return sub(r'(\d+)(\D)', lambda m: m.group(2) * int(m.group(1)), data) return decompressed
def parse(data, key): fp = BytesIO(data) build_dictionary(fp, key) records = Record.parse(fp) out = StringIO() print_records(records, fp=out) out.seek(0) if pygments is not None: print(pygments.highlight(out.read(), pygments.lexers.get_lexer_by_name('XML'), pygments.formatters.get_formatter_by_name('terminal'))) else: print(out.read())
def bsdiffParse(self): diff = StringIO(self.rleUnpack()) ctrlBlockSize, diffBlockSize, sizeAfter = self.__bsdiffParseHeader(diff) ## # The data part of the file is divded into three parts # * The control block, which contains the control part every chunk # * The diff block, which contains the diff chunks # * And the extra block, which contains the extra chunks ctrlBlock = StringIO(diff.read(ctrlBlockSize)) diffBlock = StringIO(diff.read(diffBlockSize)) extraBlock = StringIO(diff.read()) diff.close() return ctrlBlock, diffBlock, extraBlock
def test_print_items(self, bold, PL, PD): from kamaki.cli.utils import print_items, INDENT_TAB for args in product( ( 42, None, 'simple outputs', [1, 2, 3], {1: 1, 2: 2}, (3, 4), ({'k': 1, 'id': 2}, [5, 6, 7], (8, 9), '10')), (('id', 'name'), ('something', 2), ('lala', )), (False, True), (False, True)): items, title, with_enumeration, with_redundancy = args pl_counter, pd_counter = len(PL.mock_calls), len(PD.mock_calls) bold_counter, out_counter = len(bold.mock_calls), 0 out = StringIO() print_items(*args, out=out) out.seek(0) if not (isinstance(items, dict) or isinstance( items, list) or isinstance(items, tuple)): if items: self.assertEqual(out.getvalue(), '%s\n' % items) else: for i, item in enumerate(items): if with_enumeration: exp_str = '%s. ' % (i + 1) self.assertEqual(out.read(len(exp_str)), exp_str) if isinstance(item, dict): title = sorted(set(title).intersection(item)) pick = item.get if with_redundancy else item.pop header = ' '.join('%s' % pick(key) for key in title) if header: self.assertEqual( bold.mock_calls[bold_counter], call(header)) self.assertEqual(out.read(5), 'bold\n') bold_counter += 1 else: out.read(1) self.assertEqual( PD.mock_calls[pd_counter], call(item, indent=INDENT_TAB, out=out)) pd_counter += 1 elif isinstance(item, list) or isinstance(item, tuple): self.assertEqual( PL.mock_calls[pl_counter], call(item, indent=INDENT_TAB, out=out)) pl_counter += 1 else: exp_str = u' %s\n' % item self.assertEqual(out.read(len(exp_str)), exp_str)
def run_pip_main(cls, *args, **kwargs): import pip args = list(args) check_output = kwargs.pop('check_output', False) if check_output: from io import StringIO out = StringIO() sys.stdout = out try: pip.main(args) except: traceback.print_exc() finally: sys.stdout = sys.__stdout__ out.seek(0) pipdata = out.read() out.close() print(pipdata) return pipdata else: return pip.main(args)
def test_versions_and_sources_last(self): cfg = self.given_a_file_in_test_dir('buildout.cfg', '''\ [buildout] [versions] [sources] [www] [zzz] [aaa]''') output = StringIO() sort(open(cfg), output) output.seek(0) expected = '''\ [buildout] [aaa] [www] [zzz] [sources] [versions] ''' self.assertEqual(expected, output.read())
def test_exclusion_regexp(self): entries = [{'MESSAGE': 'exclude this'}, {'MESSAGE': 'message 1'}, {'MESSAGE': 'exclude that'}, {'MESSAGE': 'message 2'}] (flexmock(journal.Reader) .should_receive('get_next') .and_return(entries[0]) .and_return(entries[1]) .and_return(entries[2]) .and_return(entries[3]) .and_return({})) exclusions = [{'MESSAGE': ['/1/']}, # shouldn't exclude anything {'MESSAGE': ['/exclude th/']}, {'MESSAGE': ['/exclude/']}] formatter = EntryFormatter() jfilter = JournalFilter(journal.Reader(), [formatter], default_exclusions=exclusions) output = StringIO() jfilter.format(output) output.seek(0) lines = output.read().splitlines() assert lines == [entry['MESSAGE'] for entry in [entries[1]] + [entries[3]]] stats = jfilter.get_statistics() for stat in stats: if stat.exclusion['MESSAGE'] == ['/1/']: assert stat.hits == 0 break
class LoggerTest(unittest.TestCase): """Testcase for the Logger actor. """ def setUp(self): self._file_like = StringIO() self._logger = Logger.start(file_like=self._file_like).proxy() def tearDown(self): self._logger.stop() def _stop_logger(self): self._logger.stop() self._logger.actor_stopped.get().wait() def test_records_are_recorded(self): """Verify that logging records produces messages. """ mutation_record = MutationRecord( 'foo', 'foo.py', 'operator', {'description': 'desc', 'line_number': 3}, None) test_result = TestResult(Outcome.KILLED, 'ok') self._logger.handle_result(mutation_record, test_result) self._stop_logger() self._file_like.flush() self._file_like.seek(0) self.assertGreater(len(self._file_like.read()), 0)
def get_all_rows(es_params, request): buffer_ = StringIO() writer = csv.writer(buffer_) writer.writerow([feature for feature in es_params['features']]) ds = Datasets().activate_dataset(request.session) es_m = ds.build_manager(ES_Manager) es_m.build(es_params) es_m.set_query_parameter('size', ES_SCROLL_BATCH) features = sorted(es_params['features']) response = es_m.scroll() scroll_id = response['_scroll_id'] hits = response['hits']['hits'] while hits: process_hits(hits, features, write=True, writer=writer) buffer_.seek(0) data = buffer_.read() buffer_.seek(0) buffer_.truncate() yield data response = es_m.scroll(scroll_id=scroll_id) hits = response['hits']['hits'] scroll_id = response['_scroll_id']
def save(self, force_update=False, force_insert=False, thumb_size=(IMAGE_MAX_HEIGHT, IMAGE_MAX_WIDTH)): """Save the article. This will save thumbnail on disk and then save the model in database. """ self.slug = slugify(self.title) if has_changed(self, 'image') and self.image: # TODO : delete old image image = Image.open(self.image) if image.mode not in ('L', 'RGB'): image = image.convert('RGB') image.thumbnail(thumb_size, Image.ANTIALIAS) # save the thumbnail to memory temp_handle = StringIO() image.save(temp_handle, 'png') temp_handle.seek(0) # rewind the file # save to the thumbnail field suf = SimpleUploadedFile(os.path.split(self.image.name)[-1], temp_handle.read(), content_type='image/png') self.thumbnail.save('{}.png'.format(suf.name), suf, save=False) # save the image object super().save(force_update, force_insert) else: super().save()
def test_flush_pipes_data_between_streams(self): a = StringIO(u'food') b = StringIO() pump = io.Pump(a, b) pump.flush(3) expect(a.read(1)).to.equal('d') expect(b.getvalue()).to.equal('foo')
def render_me(element): ''' a utility to render an elment so you can see if its doing the right thing''' f = StringIO() element.render(f) f.seek(0) return f.read()
class CORATransformer: """ This class captures our understanding of the agreed format of the UKIS survey. """ _MAP_YN_10 = { 'yes': '1', 'no': '0', } _MAP_YN_01 = { 'yes': '0', 'no': '1', } _MAP_YN_21 = { 'yes': '10', 'no': '01', } _MAP_IMPORTANCE = { 'not important': '0001', 'low': '0010', 'medium': '0100', 'high': '1000', } _MAP_PROPORTIONS = { 'none': '0001', 'less than 40%': '0010', '40-90%': '0100', 'over 90%': '1000', } class _Format(enum.Enum): zeroone = re.compile("^[01]{1}$") onetwo = re.compile("^[12]{1}$") twobin = re.compile("^[0-1]{1,2}$") twodigits = re.compile("^[0-9]{1,2}$") threedigits = re.compile("^[0-9]{1,3}$") sixdigits = re.compile("^[0-9]{1,6}$") sevendigits = re.compile("^[0-9]{1,7}$") onehotfour = re.compile("^(1000|0100|0010|0001|0000)$") yesno = re.compile("^yes|no$") yesnodk = re.compile("^yes|no|(don.+t know)$") class _Processor: @staticmethod def false(q, d): return "0" @staticmethod def checkbox(q, d): return '0' if q not in d else '1' @staticmethod def checkboxtwobit(q, d): return '00' if q not in d else '10' @staticmethod def radioyn10(q, d): return '0' if q not in d else CORATransformer._MAP_YN_10.get( d[q].lower(), '0') @staticmethod def radioyn01(q, d): return '1' if q not in d else CORATransformer._MAP_YN_01.get( d[q].lower(), '1') @staticmethod def radioyn21(q, d): return '00' if q not in d else CORATransformer._MAP_YN_21.get( d[q].lower(), '00') @staticmethod def radioyndk(q, d): return '1' if d.get(q, "").lower() == "yes" else '0' @staticmethod def radioimportance(q, d): return '0000' if q not in d else CORATransformer._MAP_IMPORTANCE.get( d[q].lower(), '0000') @staticmethod def radioproportion(q, d): return '0000' if q not in d else CORATransformer._MAP_PROPORTIONS.get( d[q].lower(), '0000') @staticmethod def zeropadthree(q, d): return '' if q not in d else '{0:03d}'.format(int(d[q])) @staticmethod def zeropadtwo(q, d): return '' if q not in d else '{0:02d}'.format(int(d[q])) @staticmethod def dividebythousand(q, d): if q not in d: return '' if d[q].isdigit(): return str(int(d[q]) // 1000) return '' @staticmethod def numbertype(q, d): return '' if q not in d else d[q] @staticmethod def comment(q, d): return '0' if q not in d else '1' if len(d[q].strip()) > 0 else '0' _defn = [ (range(1, 4, 1), "0", _Format.zeroone, _Processor.false), (range(210, 250, 10), "0", _Format.zeroone, _Processor.checkbox), (range(410, 440, 10), "0", _Format.zeroone, _Processor.radioyn10), (range(2310, 2350, 10), "0", _Format.zeroone, _Processor.radioyn10), (range(1310, 1311, 1), "0", _Format.zeroone, _Processor.radioyn10), (range(2675, 2678, 1), "0", _Format.zeroone, _Processor.checkbox), (range(1410, 1411, 1), "", _Format.sixdigits, _Processor.dividebythousand), (range(1320, 1321, 1), "0", _Format.zeroone, _Processor.radioyn10), (range(1420, 1421, 1), "", _Format.sixdigits, _Processor.dividebythousand), (range(1331, 1334, 1), "0", _Format.zeroone, _Processor.checkbox), (range(1430, 1431, 1), "", _Format.sixdigits, _Processor.dividebythousand), (range(1340, 1341, 1), "0", _Format.zeroone, _Processor.radioyn10), (range(1440, 1441, 1), "", _Format.sixdigits, _Processor.dividebythousand), (range(1350, 1351, 1), "0", _Format.zeroone, _Processor.radioyn10), (range(1450, 1451, 1), "", _Format.sixdigits, _Processor.dividebythousand), (range(1360, 1361, 1), "0", _Format.zeroone, _Processor.radioyn10), (range(1460, 1461, 1), "", _Format.sixdigits, _Processor.dividebythousand), (range(1371, 1375, 1), "0", _Format.zeroone, _Processor.checkbox), (range(1470, 1471, 1), "", _Format.sixdigits, _Processor.dividebythousand), (range(510, 511, 1), "00", _Format.twobin, _Processor.radioyn21), (range(610, 640, 10), "0", _Format.zeroone, _Processor.checkbox), (range(520, 521, 1), "00", _Format.twobin, _Processor.radioyn21), (range(601, 604, 1), "0", _Format.zeroone, _Processor.checkbox), (range(710, 730, 10), "0", _Format.zeroone, _Processor.radioyn10), (range(810, 850, 10), "", _Format.threedigits, _Processor.zeropadthree), (range(900, 901, 1), "00", _Format.twobin, _Processor.radioyn21), (range(1010, 1040, 10), "0", _Format.zeroone, _Processor.checkbox), (range(1100, 1101, 1), "00", _Format.twobin, _Processor.radioyn21), (range(1510, 1540, 10), "0", _Format.zeroone, _Processor.radioyn10), (range(2657, 2668, 1), "0000", _Format.onehotfour, _Processor.radioimportance), (range(2011, 2012, 1), "0", _Format.zeroone, _Processor.checkbox), (range(2020, 2050, 10), "0", _Format.zeroone, _Processor.checkbox), (range(1210, 1212, 1), "0000", _Format.onehotfour, _Processor.radioimportance), (range(1220, 1300, 10), "0000", _Format.onehotfour, _Processor.radioimportance), (range(1212, 1214, 1), "0000", _Format.onehotfour, _Processor.radioimportance), (range(1601, 1602, 1), "0000", _Format.onehotfour, _Processor.radioimportance), (range(1620, 1621, 1), "0000", _Format.onehotfour, _Processor.radioimportance), (range(1610, 1612, 1), "0000", _Format.onehotfour, _Processor.radioimportance), (range(1631, 1633, 1), "0000", _Format.onehotfour, _Processor.radioimportance), (range(1640, 1700, 10), "0000", _Format.onehotfour, _Processor.radioimportance), (range(1811, 1815, 1), "00", _Format.twobin, _Processor.checkboxtwobit), (range(1821, 1825, 1), "00", _Format.twobin, _Processor.checkboxtwobit), (range(1881, 1885, 1), "00", _Format.twobin, _Processor.checkboxtwobit), (range(1891, 1895, 1), "00", _Format.twobin, _Processor.checkboxtwobit), (range(1841, 1845, 1), "00", _Format.twobin, _Processor.checkboxtwobit), (range(1851, 1855, 1), "00", _Format.twobin, _Processor.checkboxtwobit), (range(1861, 1865, 1), "00", _Format.twobin, _Processor.checkboxtwobit), (range(1871, 1875, 1), "00", _Format.twobin, _Processor.checkboxtwobit), (range(2650, 2657, 1), "0000", _Format.onehotfour, _Processor.radioproportion), (range(2668, 2671, 1), "0", _Format.zeroone, _Processor.radioyn10), (range(2672, 2674, 1), "0", _Format.zeroone, _Processor.radioyndk), (range(2410, 2430, 10), "", _Format.sixdigits, _Processor.dividebythousand), (range(2440, 2450, 10), "", _Format.sixdigits, _Processor.dividebythousand), (range(2510, 2530, 10), "", _Format.sevendigits, _Processor.numbertype), (range(2610, 2630, 10), "", _Format.threedigits, _Processor.zeropadthree), (range(2631, 2637, 1), "0", _Format.zeroone, _Processor.checkbox), (range(2678, 2679, 1), "0000", _Format.onehotfour, _Processor.radioimportance), (range(2700, 2701, 1), "0", _Format.zeroone, _Processor.comment), (range(2801, 2802, 1), "", _Format.threedigits, _Processor.numbertype), (range(2800, 2801, 1), "", _Format.twodigits, _Processor.zeropadtwo), (range(2900, 2901, 1), "00", _Format.twobin, _Processor.radioyn21), ] def __init__(self, logger, survey, response_data, sequence_no=1000): self._logger = logger self._survey = survey self._response = response_data self._sequence_no = sequence_no self._idbr = StringIO() self._response_json = StringIO() self._tkn = StringIO() self.image_transformer = ImageTransformer( self._logger, self._survey, self._response, CoraPdfTransformerStyle(), sequence_no=self._sequence_no, base_image_path=SDX_FTP_IMAGE_PATH) self._setup_logger() def create_zip(self): """ Create a in memory zip from a renumbered sequence""" idbr_name = self._create_idbr() tkn_name = self._create_tkn() response_io_name = self._create_response_json() self.image_transformer.zip.append( os.path.join(SDX_FTP_DATA_PATH, tkn_name), self._tkn.read()) self.image_transformer.zip.append( os.path.join(SDX_FTP_RECEIPT_PATH, idbr_name), self._idbr.read()) self.image_transformer.get_zipped_images() self.image_transformer.zip.append( os.path.join(SDX_RESPONSE_JSON_PATH, response_io_name), self._response_json.read()) self.image_transformer.zip.rewind() def get_zip(self): """Get access to the in memory zip """ self.image_transformer.zip.rewind() return self.image_transformer.zip.in_memory_zip def _create_tkn(self): data = CORATransformer._transform(self._response["data"]) output = CORATransformer._tkn_lines( surveyCode=self._response["survey_id"], ruRef=self._response["metadata"]["ru_ref"][:11], period=self._response["collection"]["period"], data=data) for row in output: self._tkn.write(row) self._tkn.write("\n") self._tkn.seek(0) tkn_name = "{0}_{1:04}".format(self._survey["survey_id"], self._sequence_no) return tkn_name def _create_idbr(self): template = env.get_template('idbr.tmpl') template_output = template.render(response=self._response) submission_date = dateutil.parser.parse(self._response['submitted_at']) submission_date_str = submission_date.strftime("%d%m") # Format is RECddMM_batchId.DAT # e.g. REC1001_30000.DAT for 10th January, batch 30000 idbr_name = "REC%s_%04d.DAT" % (submission_date_str, self._sequence_no) self._idbr.write(template_output) self._idbr.seek(0) return idbr_name def _setup_logger(self): if self._survey: if 'metadata' in self._survey: metadata = self._survey['metadata'] self._logger = self._logger.bind(user_id=metadata['user_id'], ru_ref=metadata['ru_ref']) if 'tx_id' in self._survey: self.tx_id = self._survey['tx_id'] self._logger = self._logger.bind(tx_id=self.tx_id) def _create_response_json(self): original_json_name = "%s_%04d.json" % (self._survey['survey_id'], self._sequence_no) self._response_json.write(json.dumps(self._response)) self._response_json.seek(0) return original_json_name @staticmethod def _checks(): """ Returns a dictionary mapping question ids to field formats. """ return OrderedDict([("{0:04}".format(i), check) for rng, val, check, op in CORATransformer._defn for i in rng]) @staticmethod def _ops(): """ Returns a dictionary mapping question ids to field operations. """ return OrderedDict([("{0:04}".format(i), op) for rng, val, check, op in CORATransformer._defn for i in rng]) @staticmethod def _defaults(): """ Returns a dictionary mapping question ids to default values. """ return OrderedDict([("{0:04}".format(i), val) for rng, val, check, op in CORATransformer._defn for i in rng]) @staticmethod def _transform(data): rv = CORATransformer._defaults() ops = CORATransformer._ops() # Don't know generation if any( data.get(q, "").lower().endswith("t know") for q in ("2672", "2673")): rv["2674"] = "1" else: rv["2674"] = "0" for q in rv: if q == '10001': del rv[q] # Run the processors try: op = ops[q] except KeyError: continue else: rv[q] = op(q, data) # None-of-the-above generation if not any(rv.get(k) == "1" for k in ("0410", "0420", "0430")): rv["0440"] = "1" else: rv["0440"] = "0" if not any(rv.get(k) == "1" for k in ("2668", "2669", "2670")): rv["2671"] = "1" else: rv["2671"] = "0" return rv @staticmethod def _tkn_lines(surveyCode, ruRef, period, data): pageId = "1" questionInstance = "0" return [ ":".join( (surveyCode, ruRef, pageId, period, questionInstance, q, a)) for q, a in data.items() ]
def test_dump(json, nl): buff = StringIO() json.dump([1], buff, nl=nl) buff.seek(0) nl = "\n" if nl else "" assert buff.read() == f"[1]{nl}"
TEST_FILE = pd.read_json(HERE / "FIXED_STATIONDATA.JSON") # Prepare csv for regular "downloading" test CSV_FILE = StringIO() TEST_FILE.to_csv(CSV_FILE, sep=";") CSV_FILE.seek(0) @pytest.mark.xfail @patch( "wetterdienst.dwd.observations.fileindex.create_file_list_for_climate_observations", MagicMock(return_value=[filename]), ) @patch( "wetterdienst.dwd.observations.access.download_climate_observations_data_parallel", MagicMock(return_value=[(filename, BytesIO(CSV_FILE.read().encode()))]), ) def test_collect_dwd_data_success(): """ Test for data collection """ """ 1. Scenario This scenario makes sure we take fresh data and write it to the given folder, thus we can run just another test afterwards as no old data is used """ assert collect_climate_observations_data( station_ids=[1], parameter=Parameter.CLIMATE_SUMMARY, time_resolution=TimeResolution.DAILY, period_type=PeriodType.HISTORICAL, prefer_local=False, write_file=True,
class Lexer(object): """ Simple Lexer base class. Provides basic lexer framework and helper functionality (read/peek/pushback etc) Each state is implemented using a method (lexXXXX) which should match a single token and return a (token,lexYYYY) tuple, with lexYYYY representing the next state. If token is None this is not emitted and if lexYYYY is None or the lexer reaches the end of the input stream the lexer exits. The 'parse' method returns a generator that will return tokens (the class also acts as an iterator) The default start state is 'lexStart' Input can either be a string/bytes or file object. The approach is based loosely on Rob Pike's Go lexer presentation (using generators rather than channels). >>> p = Lexer("a bcd efgh") >>> p.read() 'a' >>> p.read() ' ' >>> p.peek(3) 'bcd' >>> p.read(5) 'bcd e' >>> p.pushback('e') >>> p.read(4) 'efgh' """ escape_chars = '\\' escape = {'n': '\n', 't': '\t', 'r': '\r'} def __init__(self, f, debug=False): if hasattr(f, 'read'): self.f = f elif type(f) == str: self.f = StringIO(f) elif type(f) == bytes: self.f = StringIO(f.decode()) else: raise ValueError("Invalid input") self.debug = debug self.q = collections.deque() self.state = self.lexStart self.escaped = False self.eof = False def __iter__(self): return self.parse() def next_token(self): if self.debug: print("STATE", self.state) (tok, self.state) = self.state() return tok def parse(self): while self.state is not None and not self.eof: tok = self.next_token() if tok: yield tok def read(self, n=1): s = "" while self.q and n > 0: s += self.q.popleft() n -= 1 s += self.f.read(n) if s == '': self.eof = True if self.debug: print("Read: >%s<" % repr(s)) return s def peek(self, n=1): s = "" i = 0 while len(self.q) > i and n > 0: s += self.q[i] i += 1 n -= 1 r = self.f.read(n) if n > 0 and r == '': self.eof = True self.q.extend(r) if self.debug: print("Peek : >%s<" % repr(s + r)) return s + r def pushback(self, s): p = collections.deque(s) p.extend(self.q) self.q = p def readescaped(self): c = self.read(1) if c in self.escape_chars: self.escaped = True n = self.peek(3) if n.isdigit(): n = self.read(3) if self.debug: print("Escape: >%s<" % n) return chr(int(n, 8)) elif n[0] in 'x': x = self.read(3) if self.debug: print("Escape: >%s<" % x) return chr(int(x[1:], 16)) else: c = self.read(1) if self.debug: print("Escape: >%s<" % c) return self.escape.get(c, c) else: self.escaped = False return c def lexStart(self): return (None, None)
# print(opcode, parameters) if first_param_mode: parameters[0] = data.index( parameters[0]) if len(parameters) > 1: if second_param_mode: parameters[1] = data.index( parameters[1]) if third_param_mode: parameters[2] = data.index( parameters[2]) # print(codes[opcode].__name__, parameters) codes[opcode](data, *parameters, position) # print(opcode) position.set(position.get() + offsets[opcode]) if opcode == 4: break opcode = data[position.get()] if (opcode % 100) == 99: break if (opcode % 100) == 99: break else: continue break print(values[0]) out.seek(0) print(max(map(int, out.read().splitlines())))
class AsyncStringIO: def __init__(self, string): self.string_io = StringIO(string) async def read(self, n=-1): return self.string_io.read(n)
x = np.squeeze(v.values) y = np.asarray(v.index, dtype=np.float64) yi = np.interp(x, xp, yp) abs_diff = np.abs(y - yi) max_diff = np.max(abs_diff) if max_diff > 0.05: selected[np.where(abs_diff == max_diff)] = True selected[v.index <= 10.0] = True quantiles = list(np.squeeze(v[selected].index.values)) critical_values = list(np.squeeze(v[selected].values)) # Fix for first CV critical_values[0] = 0.0 sio.write(k + " = (") count = 0 for c, q in zip(critical_values, quantiles): sio.write("(" + "{0:0.3f}".format(q) + ", " + "{0:0.4f}".format(c) + ")") count += 1 if count % 4 == 0: sio.write(",\n " + " " * len(k)) else: sio.write(", ") sio.write(")\n") sio.write("kpss_critical_values['" + k + "'] = ") sio.write("asarray(" + k + ")") sio.write("\n") sio.seek(0) print(sio.read())
f = StringIO('Hello \n Wrold \n Goodbye') # 读内存数据 while True: s = f.readline() if s=='' : break; print(s.strip()) # StringIO操作的只能是str,如果要操作二进制数据,就需要使用BytesIO。 f = BytesIO() f.write('中文'.encode('utf-8')) # 写二进制 print(f.getvalue()) #请注意,写入的不是str,而是经过UTF-8编码的bytes。和StringIO类似,可以用一个bytes初始化BytesIO,然后,像读文件一样读取: f = BytesIO(b'\xe4\xb8\xad\xe6\x96\x87') print(f.read()) ''' ----------------------------------------------------------------------------------- part3 操作文件和目录''' print(os.name) print(os.uname()) print(os.environ) #环境变量 print(os.environ.get('PATH')) #获取某个环境变量的值 # 操作文件和目录,操作文件和目录的函数一部分放在os模块中,一部分放在os.path模块中 g = os.path.abspath('.') # 查看当前目录的绝对路径: print(g) #把两个路径合成一个时,不要直接拼字符串,而要通过os.path.join()函数,这样可以正确处理不同操作系统的路径分隔符/和\的问题 os.path.join('/home/wes/soft/pycharm/test', 'testdir') # 在某个目录下创建一个新目录,首先把新目录的完整路径表示出来 os.mkdir('/home/wes/soft/pycharm/test/testdir') # # 然后创建一个目录 os.rmdir('/home/wes/soft/pycharm/test/testdir') # 删除一个目录
class PresenterTests(TestCase): def setUp(self) -> None: self.result_output = StringIO() self.error_output = StringIO() self.renderer = TestingRepositoryRenderer() self.presenter = PresenterImpl( result_output=self.result_output, error_output=self.error_output, repository_renderer=self.renderer, ) self.repo = PrefetchedRepository( GithubRepository(owner="test", name="test"), rev="test", sha256="test", options=PrefetchOptions(), ) def test_write_to_error_output_when_presenting_prefetch_failure( self) -> None: self.presenter.present( PrefetchFailure( reason=PrefetchFailure.Reason.unable_to_calculate_sha256)) self.assertTrue(self.read_error_output()) def test_write_to_result_output_when_presenting_prefetched_repository( self) -> None: self.presenter.present(self.repo) self.assertTrue(self.read_result_output()) def test_nothing_is_written_to_result_output_when_presenting_prefetch_failure( self, ) -> None: self.presenter.present( PrefetchFailure( reason=PrefetchFailure.Reason.unable_to_calculate_sha256)) self.assertFalse(self.read_result_output()) def test_nothing_is_written_to_error_output_when_presenting_prefetched_respository( self, ) -> None: self.presenter.present(self.repo) self.assertFalse(self.read_error_output()) def test_rendered_repository_is_writted_to_result_output(self) -> None: self.presenter.present(self.repo) self.assertIn( self.renderer.render_prefetched_repository(self.repo), self.read_result_output(), ) def test_that_exit_0_is_returned_when_repository_is_rendered(self) -> None: return_code = self.presenter.present(self.repo) self.assertEqual(return_code, 0) def test_that_exit_1_is_returned_when_failure_is_rendered(self) -> None: return_code = self.presenter.present( PrefetchFailure( reason=PrefetchFailure.Reason.unable_to_calculate_sha256)) self.assertEqual(return_code, 1) def read_error_output(self) -> str: self.error_output.seek(0) return self.error_output.read() def read_result_output(self) -> str: self.result_output.seek(0) return self.result_output.read()
def _drawImageLevel2(self, image, x1, y1, x2=None, y2=None): # Postscript Level2 version try: from PIL import Image except ImportError: print('Python Imaging Library not available') return # I don't have zlib -cwl # try: # import zlib # except ImportError: # print 'zlib not available' # return ### what sort of image are we to draw if image.mode == 'L': print('found image.mode= L') imBitsPerComponent = 8 imNumComponents = 1 myimage = image elif image.mode == '1': print('found image.mode= 1') myimage = image.convert('L') imNumComponents = 1 myimage = image else: myimage = image.convert('RGB') imNumComponents = 3 imBitsPerComponent = 8 imwidth, imheight = myimage.size # print 'imwidth = %s, imheight = %s' % myimage.size if not x2: x2 = imwidth + x1 if not y2: y2 = y1 + imheight drawwidth = x2 - x1 drawheight = y2 - y1 self.code.extend([ 'gsave', '%s %s translate' % (x1, -y1 - drawheight), # need to start are lower left of image '%s %s scale' % (drawwidth, drawheight) ]) if imNumComponents == 3: self.code.append('/DeviceRGB setcolorspace') elif imNumComponents == 1: self.code.append('/DeviceGray setcolorspace') print('setting colorspace gray') # create the image dictionary self.code.append(""" << /ImageType 1 /Width %d /Height %d %% dimensions of source image /BitsPerComponent %d""" % (imwidth, imheight, imBitsPerComponent)) if imNumComponents == 1: self.code.append('/Decode [0 1]') if imNumComponents == 3: self.code.append('/Decode [0 1 0 1 0 1] %% decode color values normally') self.code.extend(['/ImageMatrix [%s 0 0 %s 0 %s]' % (imwidth, -imheight, imheight), '/DataSource currentfile /ASCIIHexDecode filter', '>> % End image dictionary', 'image']) # after image operator just need to dump image dat to file as hexstring rawimage = getattr(myimage, 'tobytes', myimage.tostring)() assert len(rawimage) == imwidth * imheight, 'Wrong amount of data for image' #compressed = zlib.compress(rawimage) # no zlib at moment hex_encoded = self._AsciiHexEncode(rawimage) # write in blocks of 78 chars per line outstream = StringIO(hex_encoded) dataline = outstream.read(78) while dataline != "": self.code.append(dataline) dataline = outstream.read(78) self.code.append('> % end of image data') # > is EOD for hex encoded filterfor clarity self.code.append('grestore') # return coordinates to normal
def dumps(self, reqs, project: Optional[RootDependency] = None, content: Optional[str] = None) -> str: if content: doc = yaml_load(content, safe=False) else: doc = dict() doc['name'] = project.name # channels if 'channels' not in doc: doc['channels'] = [] channels = set() # get channels for req in reqs: if isinstance(req.dep.repo, CondaRepo): channels.update(req.dep.repo.channels) # remove old channels for index, channel in reversed(list(enumerate(doc['channels']))): if channel not in channels: del doc['channels'][index] # add new channels for channel in channels: if channel not in doc['channels']: doc['channels'].append(channel) # add default channel if not doc['channels']: doc['channels'] = ['defaults'] # dependencies if 'dependencies' not in doc: doc['dependencies'] = [] deps = {req.name: req.version for req in reqs} # remove old deps represented = set() for index, dep in reversed(list(enumerate(doc['dependencies']))): parsed = CondaRepo.parse_req(dep) name = canonicalize_name(parsed['name']) represented.add(name) if name not in deps: # remove del doc['dependencies'][index] elif parsed.get('version', '*') != deps[name]: # update dep = name if deps[name] in ('', '*') else '{} {}'.format( name, deps[name]) doc['dependencies'][index] = dep # add new deps for name, version in sorted(deps.items()): if name in represented: continue dep = name if version in ('', '*') else '{} {}'.format(name, version) doc['dependencies'].append(dep) # drop empty section if not doc['dependencies']: del doc['dependencies'] stream = StringIO() yaml_dump(doc, stream) stream.seek(0) return stream.read()
def summary(self): s = StringIO() print(json.dumps(self.to_log_dict(), indent="\t"), file=s) s.seek(0) return s.read()
def generateR(url): notPlot = False netcdf = False data_type = "plot" try: url = json.loads(url) fileExtension = ".png" except: notPlot = True url_tail = "&" + re.findall("&(.*)", url)[0] url = re.sub("&(.*)", "", url) url = json.loads(url) fileExtension = ".csv" if 'output_format' in url: netcdf = True fileExtension = ".nc" data_type = "subset" #setup file script = StringIO() #FILE CONTENTS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ script.write("#Assigns Base Url\n\n") script.write('base_url <- "http://navigator.oceansdata.ca/api/v1.0/' + data_type + '/"\n\n\n') old_query = dict(url) #Assign Query Variables script.write("#Assignes Query Variables\n") for x in url: updated_value = re.sub("'", '"', str(url.get(x))) updated_value = re.sub("True", "true", updated_value) updated_value = re.sub("False", "false", updated_value) if isinstance(url.get(x), str): script.write(x + '= ' "'" + '"' + updated_value + '"' + "'\n") elif isinstance(url.get(x), bool): script.write(x + '= "' + updated_value + '"\n') elif isinstance(url.get(x), int): if x == 'time': formatted_date = time_query_conversion(url.get('dataset'), url.get('time')) script.write(x + '= ' + "'" + '"' + formatted_date + '"' + "'" + '\n') else: script.write(x + '= ' + updated_value + '\n') else: script.write(x + '= ' + "'" + updated_value + "'\n") script.write("\n") if notPlot == True: script.write("url_tail <- '" + url_tail + "'\n") #Assemble Query queryVariables = '' query_template = url script.write("#Assembles Query Using the Assigned Values\n") script.write("query = sprintf('") for x in url: url[x] = '%s' #query_template = re.sub(str(url.get(x)), '%s', str(query_template), 1) queryVariables += ', ' + x query_template = str(query_template) query_template = re.sub("'%s'", "%s", query_template) query_template = re.sub( "'", '"', query_template) #Replace signle quotes with double quotes script.write(query_template) #queryVariables = re.sub(',', '', queryVariables, 1) script.write("'") script.write(queryVariables) script.write(')\n\n') #Request and Save Image script.write("#Request and Save Image\n") if notPlot == False: script.write('full_url <- paste0(base_url, "?query=", query)\n') else: script.write( 'full_url <- paste0(base_url, "?query=", query, url_tail)\n') if old_query.get("type") == "drifter": script.write( 'filename = paste0("script_template_", dataset, "_", drifter, ".png")\n' ) else: script.write("#Format time to be used in file name\n") script.write("time_ = gsub(':.*','', time)\n") script.write("time_ = gsub('\\" + '"' + "'," + '"",' + "time_)\n\n") script.write('filename = paste0("script_output_", time_, "' + fileExtension + '")\n') script.write( 'download.file(full_url, filename, extra = "curl", mode = "wb")\n') # EOF FILE CONTENT ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ b = BytesIO() script.seek(0) b.write(bytes(script.read(), 'utf-8')) b.seek(0) #Hash Result (For Testing) #data = b.read() #m = hashlib.md5() #m.update(data) #print(m.hexdigest()) #b.seek(0) return b
def init_sqlite(self): """get sqlite connection giving path of database file. args: sqlite_file (str): path of sqlite database file. return: connection object """ sqlite_file = os.path.join(FLAGS.data_dir, DMOZ_SQLITE) logging.info("get dmoz sqlite connection") if os.path.isfile(sqlite_file): logging.info("DB already exists: " + sqlite_file) # conn = sqlite3.connect(sqlite_file, check_same_thread=False) # Read database to tempfile con = sqlite3.connect(sqlite_file) tempfile = StringIO() for line in con.iterdump(): tempfile.write('%s\n' % line) con.close() tempfile.seek(0) # Create a database in memory and import from tempfile conn = sqlite3.connect(":memory:", check_same_thread=False) conn.cursor().executescript(tempfile.read()) conn.commit() conn.row_factory = sqlite3.Row else: dmoz_json_path = os.path.join(FLAGS.data_dir, DMOZ_JSON) if not os.path.isfile(dmoz_json_path): raise FileNotFoundError(dmoz_json_path) dmoz = read_json(dmoz_json_path) logging.info("creating new database file: " + sqlite_file) # enable Serialized, only read is used. conn = sqlite3.connect(sqlite_file, check_same_thread=False) # Create table conn.execute('CREATE TABLE dmoz (url TEXT, category INTEGER)') # conn.execute('CREATE TABLE dmoz (url TEXT PRIMARY KEY, category INTEGER)') logging.info("created table") row_ind = 0 url_chunk = [] for cat in dmoz: for page in dmoz[cat]: # url = urlparse(page['url']) # # garentee / after netloc # path = '/' # if url.path: # path = url.path url_clean = _clean_url(page['url']) url_chunk.append((url_clean, CATEGORIES.index(cat))) row_ind += 1 if row_ind % FLAGS.insert_chunk_size == 0: # sql insert many rows # logging.debug("url_chunk: {}".format(url_chunk)) conn.executemany('INSERT INTO dmoz VALUES (?,?)', url_chunk) url_chunk = [] conn.commit() logging.info( "row {}: inset {} rows to dmoz TABLE".format( row_ind, FLAGS.insert_chunk_size)) # insert the last block if url_chunk: conn.executemany('INSERT INTO dmoz VALUES (?,?)', url_chunk) url_chunk = [] conn.commit() logging.info("row {}: inset {} rows to dmoz TABLE".format( row_ind, len(url_chunk))) # create index for url logging.info("creating url index for dmonz DB") conn.execute('CREATE INDEX url_index ON dmoz (url)') # def regexp(expr, item): # reg = re.compile(expr) # return reg.search(item) is not None # # conn.create_function("REGEXP", 2, regexp) self.conn = conn
def generatePython(url): notPlot = False netcdf = False data_type = "plot" try: url = json.loads(url) except: notPlot = True url_tail = "&" + re.findall("&(.*)", url)[0] url = re.sub("&(.*)", "", url) url = json.loads(url) fileExtension = "csv" if 'output_format' in url: netcdf = True fileExtension = "nc" data_type = "subset" #setup file script = StringIO() #FILE CONTENTS~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #HEADER--------------------- if notPlot == True or netcdf == True: script.write("import requests\n") script.write("import os\n") script.write("import shutil\n") script.write("from urllib.request import urlopen\n") script.write("from urllib.parse import urlencode\n") script.write("from contextlib import closing\n") script.write("try:\n") script.write(" from PIL import Image\n") script.write("except:\n") script.write( ' print("If you are on a Windows machine, please install PIL (imaging library) using' + " 'python -m pip install Pillow'" + ' in the Anaconda Prompt")\n') script.write(" exit()\n") script.write("import json\n") script.write("\n\n") #Set Navigators URL script.write("#Set Navigator URL\n") script.write( 'base_url = "http://navigator.oceansdata.ca/api/v1.0/%s/?"\n\n' % (data_type)) #--------------------------- #CREATE JSON OBJECT--------- script.write("#Create JSON Object\n") script.write("query = {\n") for x in url: #print(type(url.get(x))) if isinstance(url.get(x), str): script.write(' "' + x + '": "' + str(url.get(x)) + '"' + ",\n") else: if x == 'time': formatted_date = time_query_conversion(url.get('dataset'), url.get('time')) script.write(' "' + x + '": ' + "'" + formatted_date + "',\n") else: script.write(' "' + x + '": ' + str(url.get(x)) + ",\n") script.write("}\n") #--------------------------- #Assemble full request script.write('\n#Assemble full request - converts json object to url\n') if notPlot == False: script.write("url = base_url + urlencode(" + '{"query": ' + "json.dumps(query)})" + "\n") else: script.write("url = base_url + urlencode(" + '{"query": ' + "json.dumps(query)}) + '" + url_tail + "'\n") script.write("print(url)\n") #Open URL and read response if notPlot == False and netcdf == False: script.write("\n#Open URL and save response\n") script.write("with closing(urlopen(url)) as f:\n") script.write(" img = Image.open(f)\n") if url.get("type") == "drifter": script.write( ' img.save("script_template_" + str(query["dataset"]) + "_" + str(query["drifter"]) + ".png", "PNG")\n' ) else: script.write( ' img.save("script_template_" + str(query["dataset"]) + "_" + str(query["variable"]) + ".png" , "PNG")\n' ) else: script.write("\n#Open URL and save response\n") script.write("data_file = requests.get(url, stream=True)\n") script.write("dump = data_file.raw\n") script.write("location = os.path.abspath('/home/script_output.%s')\n" % (fileExtension)) script.write('with open("script_output.%s", "wb") as location:\n' % (fileExtension)) script.write(" print('Saving File')\n") script.write(" shutil.copyfileobj(dump, location)\n") script.write(" print('Done')\n") #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #CONVERT TO BytesIO TO SEND b = BytesIO() script.seek(0) b.write(bytes(script.read(), 'utf-8')) b.seek(0) return b
def _run_dump(value) -> str: output = StringIO() dump(value, to=output) output.seek(0) return output.read()
class EmbeddedSphinxShell(object): """An embedded IPython instance to run inside Sphinx""" def __init__(self, exec_lines=None): self.cout = StringIO() if exec_lines is None: exec_lines = [] # Create config object for IPython config = Config() config.HistoryManager.hist_file = ':memory:' config.InteractiveShell.autocall = False config.InteractiveShell.autoindent = False config.InteractiveShell.colors = 'NoColor' # create a profile so instance history isn't saved tmp_profile_dir = tempfile.mkdtemp(prefix='profile_') profname = 'auto_profile_sphinx_build' pdir = os.path.join(tmp_profile_dir, profname) profile = ProfileDir.create_profile_dir(pdir) # Create and initialize global ipython, but don't start its mainloop. # This will persist across different EmbededSphinxShell instances. IP = InteractiveShell.instance(config=config, profile_dir=profile) atexit.register(self.cleanup) # io.stdout redirect must be done after instantiating InteractiveShell io.stdout = self.cout io.stderr = self.cout # For debugging, so we can see normal output, use this: #from IPython.utils.io import Tee #io.stdout = Tee(self.cout, channel='stdout') # dbg #io.stderr = Tee(self.cout, channel='stderr') # dbg # Store a few parts of IPython we'll need. self.IP = IP self.user_ns = self.IP.user_ns self.user_global_ns = self.IP.user_global_ns self.input = '' self.output = '' self.tmp_profile_dir = tmp_profile_dir self.is_verbatim = False self.is_doctest = False self.is_suppress = False # Optionally, provide more detailed information to shell. # this is assigned by the SetUp method of IPythonDirective # to point at itself. # # So, you can access handy things at self.directive.state self.directive = None # on the first call to the savefig decorator, we'll import # pyplot as plt so we can make a call to the plt.gcf().savefig self._pyplot_imported = False # Prepopulate the namespace. for line in exec_lines: self.process_input_line(line, store_history=False) def cleanup(self): shutil.rmtree(self.tmp_profile_dir, ignore_errors=True) def clear_cout(self): self.cout.seek(0) self.cout.truncate(0) def process_input_line(self, line, store_history=True): """process the input, capturing stdout""" stdout = sys.stdout splitter = self.IP.input_splitter try: sys.stdout = self.cout splitter.push(line) more = splitter.push_accepts_more() if not more: source_raw = splitter.raw_reset() self.IP.run_cell(source_raw, store_history=store_history) finally: sys.stdout = stdout def process_image(self, decorator): """ # build out an image directive like # .. image:: somefile.png # :width 4in # # from an input like # savefig somefile.png width=4in """ savefig_dir = self.savefig_dir source_dir = self.source_dir saveargs = decorator.split(' ') filename = saveargs[1] # insert relative path to image file in source outfile = os.path.relpath(os.path.join(savefig_dir, filename), source_dir) imagerows = ['.. image:: %s' % outfile] for kwarg in saveargs[2:]: arg, val = kwarg.split('=') arg = arg.strip() val = val.strip() imagerows.append(' :%s: %s' % (arg, val)) image_file = os.path.basename(outfile) # only return file name image_directive = '\n'.join(imagerows) return image_file, image_directive # Callbacks for each type of token def process_input(self, data, input_prompt, lineno): """ Process data block for INPUT token. """ decorator, input, rest = data image_file = None image_directive = None is_verbatim = decorator == '@verbatim' or self.is_verbatim is_doctest = (decorator is not None and \ decorator.startswith('@doctest')) or self.is_doctest is_suppress = decorator == '@suppress' or self.is_suppress is_okexcept = decorator == '@okexcept' or self.is_okexcept is_okwarning = decorator == '@okwarning' or self.is_okwarning is_savefig = decorator is not None and \ decorator.startswith('@savefig') input_lines = input.split('\n') if len(input_lines) > 1: if input_lines[-1] != "": input_lines.append('') # make sure there's a blank line # so splitter buffer gets reset continuation = ' %s:' % ''.join(['.'] * (len(str(lineno)) + 2)) if is_savefig: image_file, image_directive = self.process_image(decorator) ret = [] is_semicolon = False # Hold the execution count, if requested to do so. if is_suppress and self.hold_count: store_history = False else: store_history = True # Note: catch_warnings is not thread safe with warnings.catch_warnings(record=True) as ws: for i, line in enumerate(input_lines): if line.endswith(';'): is_semicolon = True if i == 0: # process the first input line if is_verbatim: self.process_input_line('') self.IP.execution_count += 1 # increment it anyway else: # only submit the line in non-verbatim mode self.process_input_line(line, store_history=store_history) formatted_line = '%s %s' % (input_prompt, line) else: # process a continuation line if not is_verbatim: self.process_input_line(line, store_history=store_history) formatted_line = '%s %s' % (continuation, line) if not is_suppress: ret.append(formatted_line) if not is_suppress and len(rest.strip()) and is_verbatim: # The "rest" is the standard output of the input. This needs to be # added when in verbatim mode. If there is no "rest", then we don't # add it, as the new line will be added by the processed output. ret.append(rest) # Fetch the processed output. (This is not the submitted output.) self.cout.seek(0) processed_output = self.cout.read() if not is_suppress and not is_semicolon: # # In IPythonDirective.run, the elements of `ret` are eventually # combined such that '' entries correspond to newlines. So if # `processed_output` is equal to '', then the adding it to `ret` # ensures that there is a blank line between consecutive inputs # that have no outputs, as in: # # In [1]: x = 4 # # In [2]: x = 5 # # When there is processed output, it has a '\n' at the tail end. So # adding the output to `ret` will provide the necessary spacing # between consecutive input/output blocks, as in: # # In [1]: x # Out[1]: 5 # # In [2]: x # Out[2]: 5 # # When there is stdout from the input, it also has a '\n' at the # tail end, and so this ensures proper spacing as well. E.g.: # # In [1]: print x # 5 # # In [2]: x = 5 # # When in verbatim mode, `processed_output` is empty (because # nothing was passed to IP. Sometimes the submitted code block has # an Out[] portion and sometimes it does not. When it does not, we # need to ensure proper spacing, so we have to add '' to `ret`. # However, if there is an Out[] in the submitted code, then we do # not want to add a newline as `process_output` has stuff to add. # The difficulty is that `process_input` doesn't know if # `process_output` will be called---so it doesn't know if there is # Out[] in the code block. The requires that we include a hack in # `process_block`. See the comments there. # ret.append(processed_output) elif is_semicolon: # Make sure there is a newline after the semicolon. ret.append('') # context information filename = "Unknown" lineno = 0 if self.directive.state: filename = self.directive.state.document.current_source lineno = self.directive.state.document.current_line # output any exceptions raised during execution to stdout # unless :okexcept: has been specified. if not is_okexcept and "Traceback" in processed_output: s = "\nException in %s at block ending on line %s\n" % (filename, lineno) s += "Specify :okexcept: as an option in the ipython:: block to suppress this message\n" sys.stdout.write('\n\n>>>' + ('-' * 73)) sys.stdout.write(s) sys.stdout.write(processed_output) sys.stdout.write('<<<' + ('-' * 73) + '\n\n') # output any warning raised during execution to stdout # unless :okwarning: has been specified. if not is_okwarning: for w in ws: s = "\nWarning in %s at block ending on line %s\n" % (filename, lineno) s += "Specify :okwarning: as an option in the ipython:: block to suppress this message\n" sys.stdout.write('\n\n>>>' + ('-' * 73)) sys.stdout.write(s) sys.stdout.write(('-' * 76) + '\n') s = warnings.formatwarning(w.message, w.category, w.filename, w.lineno, w.line) sys.stdout.write(s) sys.stdout.write('<<<' + ('-' * 73) + '\n') self.cout.truncate(0) return (ret, input_lines, processed_output, is_doctest, decorator, image_file, image_directive) def process_output(self, data, output_prompt, input_lines, output, is_doctest, decorator, image_file): """ Process data block for OUTPUT token. """ # Recall: `data` is the submitted output, and `output` is the processed # output from `input_lines`. TAB = ' ' * 4 if is_doctest and output is not None: found = output # This is the processed output found = found.strip() submitted = data.strip() if self.directive is None: source = 'Unavailable' content = 'Unavailable' else: source = self.directive.state.document.current_source content = self.directive.content # Add tabs and join into a single string. content = '\n'.join([TAB + line for line in content]) # Make sure the output contains the output prompt. ind = found.find(output_prompt) if ind < 0: e = ('output does not contain output prompt\n\n' 'Document source: {0}\n\n' 'Raw content: \n{1}\n\n' 'Input line(s):\n{TAB}{2}\n\n' 'Output line(s):\n{TAB}{3}\n\n') e = e.format(source, content, '\n'.join(input_lines), repr(found), TAB=TAB) raise RuntimeError(e) found = found[len(output_prompt):].strip() # Handle the actual doctest comparison. if decorator.strip() == '@doctest': # Standard doctest if found != submitted: e = ('doctest failure\n\n' 'Document source: {0}\n\n' 'Raw content: \n{1}\n\n' 'On input line(s):\n{TAB}{2}\n\n' 'we found output:\n{TAB}{3}\n\n' 'instead of the expected:\n{TAB}{4}\n\n') e = e.format(source, content, '\n'.join(input_lines), repr(found), repr(submitted), TAB=TAB) raise RuntimeError(e) else: self.custom_doctest(decorator, input_lines, found, submitted) # When in verbatim mode, this holds additional submitted output # to be written in the final Sphinx output. # https://github.com/ipython/ipython/issues/5776 out_data = [] is_verbatim = decorator == '@verbatim' or self.is_verbatim if is_verbatim and data.strip(): # Note that `ret` in `process_block` has '' as its last element if # the code block was in verbatim mode. So if there is no submitted # output, then we will have proper spacing only if we do not add # an additional '' to `out_data`. This is why we condition on # `and data.strip()`. # The submitted output has no output prompt. If we want the # prompt and the code to appear, we need to join them now # instead of adding them separately---as this would create an # undesired newline. How we do this ultimately depends on the # format of the output regex. I'll do what works for the default # prompt for now, and we might have to adjust if it doesn't work # in other cases. Finally, the submitted output does not have # a trailing newline, so we must add it manually. out_data.append("{0} {1}\n".format(output_prompt, data)) return out_data def process_comment(self, data): """Process data fPblock for COMMENT token.""" if not self.is_suppress: return [data] def save_image(self, image_file): """ Saves the image file to disk. """ self.ensure_pyplot() command = 'plt.gcf().savefig("%s")' % image_file #print 'SAVEFIG', command # dbg self.process_input_line('bookmark ipy_thisdir', store_history=False) self.process_input_line('cd -b ipy_savedir', store_history=False) self.process_input_line(command, store_history=False) self.process_input_line('cd -b ipy_thisdir', store_history=False) self.process_input_line('bookmark -d ipy_thisdir', store_history=False) self.clear_cout() def process_block(self, block): """ process block from the block_parser and return a list of processed lines """ ret = [] output = None input_lines = None lineno = self.IP.execution_count input_prompt = self.promptin % lineno output_prompt = self.promptout % lineno image_file = None image_directive = None found_input = False for token, data in block: if token == COMMENT: out_data = self.process_comment(data) elif token == INPUT: found_input = True (out_data, input_lines, output, is_doctest, decorator, image_file, image_directive) = \ self.process_input(data, input_prompt, lineno) elif token == OUTPUT: if not found_input: TAB = ' ' * 4 linenumber = 0 source = 'Unavailable' content = 'Unavailable' if self.directive: linenumber = self.directive.state.document.current_line source = self.directive.state.document.current_source content = self.directive.content # Add tabs and join into a single string. content = '\n'.join([TAB + line for line in content]) e = ('\n\nInvalid block: Block contains an output prompt ' 'without an input prompt.\n\n' 'Document source: {0}\n\n' 'Content begins at line {1}: \n\n{2}\n\n' 'Problematic block within content: \n\n{TAB}{3}\n\n') e = e.format(source, linenumber, content, block, TAB=TAB) # Write, rather than include in exception, since Sphinx # will truncate tracebacks. sys.stdout.write(e) raise RuntimeError('An invalid block was detected.') out_data = \ self.process_output(data, output_prompt, input_lines, output, is_doctest, decorator, image_file) if out_data: # Then there was user submitted output in verbatim mode. # We need to remove the last element of `ret` that was # added in `process_input`, as it is '' and would introduce # an undesirable newline. assert (ret[-1] == '') del ret[-1] if out_data: ret.extend(out_data) # save the image files if image_file is not None: self.save_image(image_file) return ret, image_directive def ensure_pyplot(self): """ Ensures that pyplot has been imported into the embedded IPython shell. Also, makes sure to set the backend appropriately if not set already. """ # We are here if the @figure pseudo decorator was used. Thus, it's # possible that we could be here even if python_mplbackend were set to # `None`. That's also strange and perhaps worthy of raising an # exception, but for now, we just set the backend to 'agg'. if not self._pyplot_imported: if 'matplotlib.backends' not in sys.modules: # Then ipython_matplotlib was set to None but there was a # call to the @figure decorator (and ipython_execlines did # not set a backend). #raise Exception("No backend was set, but @figure was used!") import matplotlib matplotlib.use('agg') # Always import pyplot into embedded shell. self.process_input_line('import matplotlib.pyplot as plt', store_history=False) self._pyplot_imported = True def process_pure_python(self, content): """ content is a list of strings. it is unedited directive content This runs it line by line in the InteractiveShell, prepends prompts as needed capturing stderr and stdout, then returns the content as a list as if it were ipython code """ output = [] savefig = False # keep up with this to clear figure multiline = False # to handle line continuation multiline_start = None fmtin = self.promptin ct = 0 for lineno, line in enumerate(content): line_stripped = line.strip() if not len(line): output.append(line) continue # handle decorators if line_stripped.startswith('@'): output.extend([line]) if 'savefig' in line: savefig = True # and need to clear figure continue # handle comments if line_stripped.startswith('#'): output.extend([line]) continue # deal with lines checking for multiline continuation = u' %s:' % ''.join(['.'] * (len(str(ct)) + 2)) if not multiline: modified = u"%s %s" % (fmtin % ct, line_stripped) output.append(modified) ct += 1 try: ast.parse(line_stripped) output.append(u'') except Exception: # on a multiline multiline = True multiline_start = lineno else: # still on a multiline modified = u'%s %s' % (continuation, line) output.append(modified) # if the next line is indented, it should be part of multiline if len(content) > lineno + 1: nextline = content[lineno + 1] if len(nextline) - len(nextline.lstrip()) > 3: continue try: mod = ast.parse('\n'.join(content[multiline_start:lineno + 1])) if isinstance(mod.body[0], ast.FunctionDef): # check to see if we have the whole function for element in mod.body[0].body: if isinstance(element, ast.Return): multiline = False else: output.append(u'') multiline = False except Exception: pass if savefig: # clear figure if plotted self.ensure_pyplot() self.process_input_line('plt.clf()', store_history=False) self.clear_cout() savefig = False return output def custom_doctest(self, decorator, input_lines, found, submitted): """ Perform a specialized doctest. """ from .custom_doctests import doctests args = decorator.split() doctest_type = args[1] if doctest_type in doctests: doctests[doctest_type](self, args, input_lines, found, submitted) else: e = "Invalid option to @doctest: {0}".format(doctest_type) raise Exception(e)
class CodeEval(Cog): """Owner and admin feature that evaluates code and returns the result to the channel.""" def __init__(self, bot: Bot): self.bot = bot self.env = {} self.ln = 0 self.stdout = StringIO() self.interpreter = Interpreter(bot) def _format(self, inp: str, out: Any) -> Tuple[str, Optional[discord.Embed]]: """Format the eval output into a string & attempt to format it into an Embed.""" self._ = out res = "" # Erase temp input we made if inp.startswith("_ = "): inp = inp[4:] # Get all non-empty lines lines = [line for line in inp.split("\n") if line.strip()] if len(lines) != 1: lines += [""] # Create the input dialog for i, line in enumerate(lines): if i == 0: # Start dialog start = f"In [{self.ln}]: " else: # Indent the 3 dots correctly; # Normally, it's something like # In [X]: # ...: # # But if it's # In [XX]: # ...: # # You can see it doesn't look right. # This code simply indents the dots # far enough to align them. # we first `str()` the line number # then we get the length # and use `str.rjust()` # to indent it. start = "...: ".rjust(len(str(self.ln)) + 7) if i == len(lines) - 2: if line.startswith("return"): line = line[6:].strip() # Combine everything res += (start + line + "\n") self.stdout.seek(0) text = self.stdout.read() self.stdout.close() self.stdout = StringIO() if text: res += (text + "\n") if out is None: # No output, return the input statement return (res, None) res += f"Out[{self.ln}]: " if isinstance(out, discord.Embed): # We made an embed? Send that as embed res += "<Embed>" res = (res, out) else: if (isinstance(out, str) and out.startswith("Traceback (most recent call last):\n")): # Leave out the traceback message out = "\n" + "\n".join(out.split("\n")[1:]) if isinstance(out, str): pretty = out else: pretty = pprint.pformat(out, compact=True, width=60) if pretty != str(out): # We're using the pretty version, start on the next line res += "\n" if pretty.count("\n") > 20: # Text too long, shorten li = pretty.split("\n") pretty = ( "\n".join(li[:3]) # First 3 lines + "\n ...\n" # Ellipsis to indicate removed lines + "\n".join(li[-3:])) # last 3 lines # Add the output res += pretty res = (res, None) return res # Return (text, embed) async def _eval(self, ctx: Context, code: str) -> Optional[discord.Message]: """Eval the input code string & send an embed to the invoking context.""" self.ln += 1 if code.startswith("exit"): self.ln = 0 self.env = {} return await ctx.send("```Reset history!```") env = { "message": ctx.message, "author": ctx.message.author, "channel": ctx.channel, "guild": ctx.guild, "ctx": ctx, "self": self, "bot": self.bot, "inspect": inspect, "discord": discord, "contextlib": contextlib } self.env.update(env) # Ignore this code, it works _code = """ async def func(): # (None,) -> Any try: with contextlib.redirect_stdout(self.stdout): {0} if '_' in locals(): if inspect.isawaitable(_): _ = await _ return _ finally: self.env.update(locals()) """.format(textwrap.indent(code, ' ')) try: exec(_code, self.env) # noqa: B102,S102 func = self.env['func'] res = await func() except Exception: res = traceback.format_exc() out, embed = self._format(code, res) await ctx.send(f"```py\n{out}```", embed=embed) @group(name='internal', aliases=('int', )) @with_role(Roles.owner, Roles.admin) async def internal_group(self, ctx: Context) -> None: """Internal commands. Top secret!""" if not ctx.invoked_subcommand: await ctx.invoke(self.bot.get_command("help"), "internal") @internal_group.command(name='eval', aliases=('e', )) @with_role(Roles.admin, Roles.owner) async def eval(self, ctx: Context, *, code: str) -> None: """Run eval in a REPL-like format.""" code = code.strip("`") if re.match('py(thon)?\n', code): code = "\n".join(code.split("\n")[1:]) if not re.search( # Check if it's an expression r"^(return|import|for|while|def|class|" r"from|exit|[a-zA-Z0-9]+\s*=)", code, re.M) and len( code.split("\n")) == 1: code = "_ = " + code await self._eval(ctx, code)
def dumps(investigation): # build MSI section metadata_DF = pd.DataFrame( columns=("Submission Title", "Submission Identifier", "Submission Description", "Submission Version", "Submission Reference Layer", "Submission Release Date", "Submission Update Date")) iversion_hits = [ x for x in investigation.comments if x.name == "Submission Version" ] if len(iversion_hits) == 1: investigation_version = iversion_hits[0].value else: investigation_version = "" ireference_layer_hits = [ x for x in investigation.comments if x.name == "Submission Reference Layer" ] if len(ireference_layer_hits) == 1: investigation_reference_layer = ireference_layer_hits[0].value else: investigation_reference_layer = "" iversion_update_date = [ x for x in investigation.comments if x.name == "Submission Update Date" ] if len(iversion_update_date) == 1: investigation_update_date = iversion_update_date[0].value else: investigation_update_date = "" metadata_DF.loc[0] = [ investigation.title, investigation.identifier, investigation.description, investigation_version, investigation_reference_layer, investigation.submission_date, investigation_update_date ] org_DF = pd.DataFrame(columns=("Organization Name", "Organization Address", "Organization URI", "Organization Email", "Organization Role")) org_name_hits = [ x for x in investigation.comments if x.name.startswith("Organization Name") ] org_address_hits = [ x for x in investigation.comments if x.name.startswith("Organization Address") ] org_uri_hits = [ x for x in investigation.comments if x.name.startswith("Organization URI") ] org_email_hits = [ x for x in investigation.comments if x.name.startswith("Organization Email") ] org_role_hits = [ x for x in investigation.comments if x.name.startswith("Organization Role") ] for i, org_name in enumerate(org_name_hits): try: org_name = org_name_hits[i].value except IndexError: org_name = "" try: org_address = org_address_hits[i].value except IndexError: org_address = "" try: org_uri = org_uri_hits[i].value except IndexError: org_uri = "" try: org_email = org_email_hits[i].value except IndexError: org_email = "" try: org_role = org_role_hits[i].value except IndexError: org_role = "" org_DF.loc[i] = [org_name, org_address, org_uri, org_email, org_role] people_DF = pd.DataFrame(columns=("Person Last Name", "Person Initials", "Person First Name", "Person Email", "Person Role")) for i, contact in enumerate(investigation.contacts): if len(contact.roles) == 1: role = contact.roles[0].term else: role = "" people_DF.loc[i] = [ contact.last_name, contact.mid_initials, contact.first_name, contact.email, role ] term_sources_DF = pd.DataFrame(columns=("Term Source Name", "Term Source URI", "Term Source Version")) for i, term_source in enumerate(investigation.ontology_source_references): term_sources_DF.loc[i] = [ term_source.name, term_source.file, term_source.version ] msi_DF = pd.concat([metadata_DF, org_DF, people_DF, term_sources_DF], axis=1) msi_DF = msi_DF.set_index("Submission Title").T msi_DF = msi_DF.replace('', np.nan) msi_memf = StringIO() msi_DF.to_csv(path_or_buf=msi_memf, index=True, sep='\t', encoding='utf-8', index_label="Submission Title") msi_memf.seek(0) scd_DF = pd.DataFrame(columns=("Sample Name", "Sample Accession", "Sample Description", "Derived From", "Group Name", "Group Accession")) all_samples = [] for study in investigation.studies: all_samples += study.materials['sources'] all_samples += study.materials['samples'] all_samples = list(set(all_samples)) pbar = ProgressBar(min_value=0, max_value=len(all_samples), widgets=[ 'Writing {} samples: '.format(len(all_samples)), SimpleProgress(), Bar(left=" |", right="| "), ETA() ]).start() for i, s in pbar(enumerate(all_samples)): derived_from = "" if isinstance(s, Sample) and s.derives_from is not None: if len(s.derives_from) == 1: derived_from_obj = s.derives_from[0] derives_from_accession_hits = [ x for x in derived_from_obj.characteristics if x.category.term == "Sample Accession" ] if len(derives_from_accession_hits) == 1: derived_from = derives_from_accession_hits[0].value else: print( "WARNING! No Sample Accession available so referencing Derived From relation using Sample Name \"{}\" instead" .format(derived_from_obj.name)) derived_from = derived_from_obj.name sample_accession_hits = [ x for x in s.characteristics if x.category.term == "Sample Accession" ] if len(sample_accession_hits) == 1: sample_accession = sample_accession_hits[0].value else: sample_accession = "" sample_description_hits = [ x for x in s.characteristics if x.category.term == "Sample Description" ] if len(sample_description_hits) == 1: sample_description = sample_description_hits[0].value else: sample_description = "" if isinstance(s, Sample): group_name_hits = [ x for x in s.factor_values if x.factor_name.name == "Group Name" ] if len(group_name_hits) == 1: group_name = group_name_hits[0].value else: group_name = "" group_accession_hits = [ x for x in s.factor_values if x.factor_name.name == "Group Accession" ] if len(group_accession_hits) == 1: group_accession = group_accession_hits[0].value else: group_accession = "" else: group_name_hits = [ x for x in s.characteristics if x.category.term == "Group Name" ] if len(group_name_hits) == 1: group_name = group_name_hits[0].value else: group_name = "" group_accession_hits = [ x for x in s.characteristics if x.category.term == "Group Accession" ] if len(group_accession_hits) == 1: group_accession = group_accession_hits[0].value else: group_accession = "" scd_DF.loc[i, "Sample Name"] = s.name scd_DF.loc[i, "Sample Accession"] = sample_accession scd_DF.loc[i, "Sample Description"] = sample_description scd_DF.loc[i, "Derived From"] = derived_from scd_DF.loc[i, "Group Name"] = group_name scd_DF.loc[i, "Group Accession"] = group_accession characteristics = [ x for x in s.characteristics if x.category.term not in ["Sample Description", "Derived From", "Sample Accession"] ] for characteristic in characteristics: characteristic_label = "Characteristic[{}]".format( characteristic.category.term) if characteristic_label not in scd_DF.columns: scd_DF[characteristic_label] = "" for val_col in get_value_columns(characteristic_label, characteristic): scd_DF[val_col] = "" if isinstance(characteristic.value, (int, float)) and characteristic.unit: if isinstance(characteristic.unit, OntologyAnnotation): scd_DF.loc[i, characteristic_label] = characteristic.value scd_DF.loc[i, characteristic_label + ".Unit"] = characteristic.unit.term scd_DF.loc[i, characteristic_label + ".Unit.Term Source REF"]\ = characteristic.unit.term_source.name if characteristic.unit.term_source else "" scd_DF.loc[i, characteristic_label + ".Unit.Term Accession Number"] = \ characteristic.unit.term_accession else: scd_DF.loc[i, characteristic_label] = characteristic.value scd_DF.loc[i, characteristic_label + ".Unit"] = characteristic.unit elif isinstance(characteristic.value, OntologyAnnotation): scd_DF.loc[i, characteristic_label] = characteristic.value.term scd_DF.loc[i, characteristic_label + ".Term Source REF"] = \ characteristic.value.term_source.name if characteristic.value.term_source else "" scd_DF.loc[i, characteristic_label + ".Term Accession Number"] = \ characteristic.value.term_accession else: scd_DF.loc[i, characteristic_label] = characteristic.value scd_DF = scd_DF.replace('', np.nan) columns = list(scd_DF.columns) for i, col in enumerate(columns): if col.endswith("Term Source REF"): columns[i] = "Term Source REF" elif col.endswith("Term Accession Number"): columns[i] = "Term Source ID" elif col.endswith("Unit"): columns[i] = "Unit" scd_DF.columns = columns scd_memf = StringIO() scd_DF.to_csv(path_or_buf=scd_memf, index=False, sep='\t', encoding='utf-8') scd_memf.seek(0) sampletab_memf = StringIO() sampletab_memf.write("[MSI]\n") for line in msi_memf: sampletab_memf.write(line.rstrip() + '\n') sampletab_memf.write("[SCD]\n") for line in scd_memf: sampletab_memf.write(line.rstrip() + '\n') sampletab_memf.seek(0) return sampletab_memf.read()
def drawInlineImage(self, image, x, y, width=None, height=None): """Draw a PIL Image into the specified rectangle. If width and height are omitted, they are calculated from the image size. Also allow file names as well as images. This allows a caching mechanism""" # print "drawInlineImage: x=%s, y=%s, width = %s, height=%s " % (x,y, width, height) try: from PIL import Image except ImportError: print('Python Imaging Library not available') return try: import zlib except ImportError: print('zlib not available') return self._currentPageHasImages = 1 if type(image) == StringType: if os.path.splitext(image)[1] in ['.jpg', '.JPG']: #directly process JPEG files #open file, needs some error handling!! imageFile = open(image, 'rb') info = self.readJPEGInfo(imageFile) imgwidth, imgheight = info[0], info[1] if info[2] == 1: colorSpace = 'DeviceGray' elif info[2] == 3: colorSpace = 'DeviceRGB' else: #maybe should generate an error, is this right for CMYK? colorSpace = 'DeviceCMYK' imageFile.seek(0) #reset file pointer imagedata = [] imagedata.append('BI') # begin image # this describes what is in the image itself imagedata.append('/Width %0.4f /Height %0.4f' % (info[0], info[1])) imagedata.append('/BitsPerComponent 8') imagedata.append('/ColorSpace /%s' % colorSpace) imagedata.append('/Filter [ /ASCII85Decode /DCTDecode]') imagedata.append('ID') #write in blocks of (??) 60 characters per line to a list compressed = imageFile.read() encoded = pdfutils._AsciiBase85Encode(compressed) outstream = StringIO(encoded) dataline = outstream.read(60) while dataline != "": imagedata.append(dataline) dataline = outstream.read(60) imagedata.append('EI') else: if not pdfutils.cachedImageExists(image): pdfutils.cacheImageFile(image) #now we have one cached, slurp it in cachedname = os.path.splitext(image)[0] + '.a85' imagedata = open(cachedname, 'rb').readlines() #trim off newlines... imagedata = map(string.strip, imagedata) #parse line two for width, height words = string.split(imagedata[1]) imgwidth = string.atoi(words[1]) imgheight = string.atoi(words[3]) else: #PIL Image #work out all dimensions myimage = image.convert('RGB') imgwidth, imgheight = myimage.size imagedata = [] imagedata.append('BI') # begin image # this describes what is in the image itself imagedata.append( '/W %0.4f /H %0.4f /BPC 8 /CS /RGB /F [/A85 /Fl]' % (imgwidth, imgheight)) imagedata.append('ID') #use a flate filter and Ascii Base 85 to compress raw = getattr(myimage, 'tobytes', myimage.tostring)() assert len( raw) == imgwidth * imgheight, "Wrong amount of data for image" compressed = zlib.compress(raw) #this bit is very fast... encoded = pdfutils._AsciiBase85Encode( compressed) #...sadly this isn't #write in blocks of (??) 60 characters per line to a list outstream = StringIO(encoded) dataline = outstream.read(60) while dataline != "": imagedata.append(dataline) dataline = outstream.read(60) imagedata.append('EI') #now build the PDF for the image. if not width: width = imgwidth if not height: height = imgheight # this says where and how big to draw it #self._code.append('ET') #self._code.append('q %0.4f 0 0 %0.4f %0.4f %0.4f cm' % (width, height, x, y+height)) if self.bottomup: self._code.append('q %0.4f 0 0 %0.4f %0.4f %0.4f cm' % (width, height, x, y)) else: # multiply height by (-1) to overcome flip in coordinate system -cwl self._code.append('q %0.4f 0 0 %0.4f %0.4f %0.4f cm' % (width, -height, x, y + height)) self._code.extend(imagedata) self._code.append('Q')
class Stdout: """PhpSploit framework's dedicated standard output wrapper, supplying some enhancements, such as pattern coloration and back logging. The 'backlog' argument is defaultly set to False, it can be enabled at initialisation if set to True, or enabled later setting the instance's backlog attribute to an empty string. NOTE: See module's help for more informations. """ def __init__(self, outfile=sys.__stdout__, backlog=False): # get original stdout self._orig_outfile = outfile # just in case we wrap at runtime o the future, # as we did with `colorama_wrapper` in the past self.outfile = self._orig_outfile # handle back logging self._backlog = StringIO() if backlog: self._has_backlog = True else: self._has_backlog = False # are colors supported ? self._has_colors = ui.output.colors() self._write_lock = False def __del__(self): """Restore the original sys.stdout on Wrapper deletion""" self._backlog.close() # dirty hack when used before argparse on main file... try: sys.stdout = self._orig_outfile except: pass def __getattr__(self, obj): """Fallback to original stdout objects for undefined methods""" return getattr(self._orig_outfile, obj) def _writeLn(self, line): """Process individual line morphing, and write it""" # Process per platform newline transformation if line.endswith('\r\n'): line = line[:-2] + os.linesep elif line.endswith('\n'): line = line[:-1] + os.linesep # special case: debug tag is only printed if VERBOSITY is True # NOTE: considering that the python print() function does another # write() to add line separator, we need a self._write_lock # canary to block it if the previous message display aborted. from core import session if line.startswith("[#] ") and not session.Conf.VERBOSITY(): self._write_lock = True return if self._write_lock: self._write_lock = False if line == os.linesep: return line = process_tags(line) # handle tagged lines coloration # Write line to stdout, and it's decolorized version on backlog # if standard output is not a tty, decolorize anything. if self._has_backlog: self._backlog.write(decolorize(line)) if not self._has_colors: line = decolorize(line) try: self.outfile.write(line) except UnicodeEncodeError: buf = encoding.encode(line) self.outfile.buffer.write(buf) def write(self, string): """Write the given string to stdout""" for line in string.splitlines(1): self._writeLn(line) @property def backlog(self): """A dedicated stdout back logging buffer""" if self._has_backlog: self._backlog.seek(0) return self._backlog.read() else: raise AttributeError() @backlog.setter def backlog(self, value): """Setting backlog's value to None or False disables it, While giving any other value resets the backlog buffer. If a non empty string is given, backlog takes it as new value """ del self.backlog if not (value is False or value is None): self._has_backlog = True if type(value) == str: self._backlog.write(decolorize(value)) @backlog.deleter def backlog(self): """Flush backlog buffer and mark it as disabled on deletion""" self._backlog.truncate(0) self._backlog.seek(0) self._has_backlog = False
def _write_summary_log(self, upgrade_id: str, requirements: UpgradeRequirements): try: Path(self.LOGS_DIR).mkdir(parents=True, exist_ok=True) summary_text = StringIO() summary_text.write('Upgrade summary ( id: {} )'.format(upgrade_id)) if requirements.cannot_upgrade: summary_text.write('\nCannot upgrade:') for dep in requirements.cannot_upgrade: type_label = self.i18n.get( 'gem.{}.type.{}.label'.format( dep.pkg.gem_name, dep.pkg.get_type().lower()), dep.pkg.get_type().capitalize()) summary_text.write( '\n * Type:{}\tName: {}\tVersion: {}\tReason: {}'. format(type_label, dep.pkg.name, dep.pkg.version if dep.pkg.version else '?', dep.reason if dep.reason else '?')) summary_text.write('\n') if requirements.to_remove: summary_text.write('\nMust be removed:') for dep in requirements.to_remove: type_label = self.i18n.get( 'gem.{}.type.{}.label'.format( dep.pkg.gem_name, dep.pkg.get_type().lower()), dep.pkg.get_type().capitalize()) summary_text.write( '\n * Type:{}\tName: {}\tVersion: {}\tReason: {}'. format(type_label, dep.pkg.name, dep.pkg.version if dep.pkg.version else '?', dep.reason if dep.reason else '?')) summary_text.write('\n') if requirements.to_install: summary_text.write('\nMust be installed:') for dep in requirements.to_install: type_label = self.i18n.get( 'gem.{}.type.{}.label'.format( dep.pkg.gem_name, dep.pkg.get_type().lower()), dep.pkg.get_type().capitalize()) summary_text.write( '\n * Type:{}\tName: {}\tVersion: {}\tReason: {}'. format(type_label, dep.pkg.name, dep.pkg.version if dep.pkg.version else '?', dep.reason if dep.reason else '?')) summary_text.write('\n') if requirements.to_upgrade: summary_text.write('\nWill be upgraded:') for dep in requirements.to_upgrade: type_label = self.i18n.get( 'gem.{}.type.{}.label'.format( dep.pkg.gem_name, dep.pkg.get_type().lower()), dep.pkg.get_type().capitalize()) summary_text.write( '\n * Type:{}\tName: {}\tVersion:{} \tNew version: {}'. format( type_label, dep.pkg.name, dep.pkg.version if dep.pkg.version else '?', dep.pkg.latest_version if dep.pkg.latest_version else '?')) summary_text.write('\n') summary_text.seek(0) with open(self.SUMMARY_FILE.format(upgrade_id), 'w+') as f: f.write(summary_text.read()) except: traceback.print_exc()
def _build_config_string(self, config): sfile = StringIO() config.write(sfile) sfile.seek(0) return sfile.read().strip()
# 获得完整f print(f.getvalue()) # >>> hello world! # 读取StringIO f = StringIO("Hello!\nHi!\nGoodbye!") while True: s = f.readline() if s == "": break print(s.strip()) # >>> # Hello! # Hi! # Goodbye! # StringIO操作的只能是str, 如果要操作二进制数据, 就需要使用BytesIO # BytesIO实现了在内存中读写bytes, 我们创建一个BytesIO, 然后写入一些bytes from io import BytesIO f = BytesIO() f.write("中文".encode("utf-8")) print(f.getvalue()) # >>> b"\xe4\xb8\xad\xe6\x96\x87" # 请注意, 写入的不是str, 而是经过UTF-8编码的bytes from io import BytesIO f = BytesIO(b"\xe4\xb8\xad\xe6\x96\x87") print(f.read().decode("utf-8")) # >>> 中文
def stream(nead_version, hashed_lines, model_class, display_values, null_value, start, end, dict_fields, **kwargs): # If kwargs 'start' and 'end' passed in URL validate and assign to dict_timestamps dict_timestamps = {} if '' not in [start, end]: dict_timestamps = get_timestamp_iso_range_day_dict(start, end) # Create buffer_ and writer objects buffer_ = StringIO() writer = csv.writer(buffer_, lineterminator="\n") # Check if values passed for 'nead_version' and 'hashed_lines' # If True: Write version and hash_lines to buffer_ if len(nead_version) > 0 and len(hashed_lines) > 0: buffer_.writelines(nead_version) buffer_.writelines(hashed_lines) # Else: Write 'display_values' to buffer_ else: buffer_.writelines(','.join(display_values) + '\n') # Generator expressions to write each row in the queryset by calculating each row as needed and not all at once # Write values that are null in database as the value assigned to 'null_value' # Check if 'dict_fields' passed, if so stream aggregate daily data if len(dict_fields) > 0: queryset = model_class.objects \ .values_list('day') \ .annotate(**dict_fields) \ .filter(**dict_timestamps) \ .order_by('timestamp_first') \ .iterator() for row in queryset: # Call write_row write_row(writer, null_value, row) # Yield data (row from database) buffer_.seek(0) data = buffer_.read() buffer_.seek(0) buffer_.truncate() yield data # Elif kwargs 'start' and 'end' passed then apply timestamps filter elif len(dict_timestamps) > 0: queryset = model_class.objects \ .values_list(*display_values) \ .filter(**dict_timestamps) \ .order_by('timestamp_iso') \ .iterator() for row in queryset: # Call write_row write_row(writer, null_value, row) # Yield data (row from database) buffer_.seek(0) data = buffer_.read() buffer_.seek(0) buffer_.truncate() yield data # Elif retrieve all data currently in database table if 'display_values' passed and 'start' and 'end' are not passed elif len(display_values) > 0: queryset = model_class.objects \ .values_list(*display_values) \ .order_by('timestamp_iso') \ .iterator() for row in queryset: # Call write_row write_row(writer, null_value, row) # Yield data (row from database) buffer_.seek(0) data = buffer_.read() buffer_.seek(0) buffer_.truncate() yield data else: raise FieldError( "ERROR (stream.py) 'display_values' not passed in API call")
def _parse_entries(src): entries = [] state = EXPECT_ENTRY for lineno, line in enumerate(src.decode('utf-8').splitlines()): # empty lines are skipped (but also terminate the notes section) sline = line.strip() if not sline or sline.startswith('#'): state = EXPECT_ENTRY continue # non-empty line with leading spaces is interpreted as a notes line if line[0] in [' ', '\t']: if state != EXPECT_ENTRY_OR_NOTES: raise SyntaxError(lineno, line, state) # add line of notes notes = entries[-1].notes if notes: notes += "\n" notes += sline entries[-1] = entries[-1]._replace(notes=notes) continue # parse line using shlex sio = StringIO(line) lexer = shlex(sio, posix=True) lexer.whitespace_split = True # otherwise, parse as an entry try: key = lexer.get_token() except ValueError as e: raise SyntaxError(lineno, line, str(e)) if not key: raise SyntaxError(lineno, line, state) key = key.rstrip(':') try: user = lexer.get_token() except ValueError as e: raise SyntaxError(lineno, line, str(e)) if not user: raise SyntaxError(lineno, line, state) user = user try: password = lexer.get_token() except ValueError as e: raise SyntaxError(lineno, line, str(e)) if not password: password = user user = notes = '' else: password = password notes = sio.read().strip() entries.append(Entry(key, user, password, notes)) state = EXPECT_ENTRY_OR_NOTES return entries
from pylint import epylint as lint from io import StringIO import numpy (pylint_stdout, pylint_stderr) = lint.py_run('test.py', return_std=True) print(StringIO.read(pylint_stdout))
""" StringIO ATENÇÂO: Para ler ou escrever dados de um arquivo do sistema operacional o software precisa ter permissão: - Permissão de Leitura -> Para ler o arquivo. - Permissão de Escrita -> Para Escrever no arquivo StringIO -> Utilizado para ler e criar arquivos em memória. """ # Primeiro fazemos o import from io import StringIO mensagem = 'Esta é apenas uma string normal' # Podemos criar um arquivo em memoria ja com uma string inserida ou mesmo vazio # para inserirmos texto depois arquivo = StringIO(mensagem) print(arquivo.read()) arquivo.write(' Outro Texto') arquivo.seek(0) print(arquivo.read())
def _drawImageLevel1(self, image, x1, y1, x2=None, y2=None, **kwargs): # Postscript Level1 version available for fallback mode when Level2 doesn't work """drawImage(self,image,x1,y1,x2=None,y2=None) : If x2 and y2 are omitted, they are calculated from image size. (x1,y1) is upper left of image, (x2,y2) is lower right of image in piddle coordinates.""" try: from PIL import Image except ImportError: print('Python Imaging Library not available') return # For now let's start with 24 bit RGB images (following piddlePDF again) print("Trying to drawImage in piddlePS") component_depth = 8 myimage = image.convert('RGB') imgwidth, imgheight = myimage.size if not x2: x2 = imgwidth + x1 if not y2: y2 = y1 + imgheight drawwidth = x2 - x1 drawheight = y2 - y1 print('Image size (%d, %d); Draw size (%d, %d)' % (imgwidth, imgheight, drawwidth, drawheight)) # now I need to tell postscript how big image is # "image operators assume that they receive sample data from # their data source in x-axis major index order. The coordinate # of the lower-left corner of the first sample is (0,0), of the # second (1,0) and so on" -PS2 ref manual p. 215 # # The ImageMatrix maps unit square of user space to boundary of the source image # # The CurrentTransformationMatrix (CTM) maps the unit square of # user space to the rect...on the page that is to receive the # image. A common ImageMatrix is [width 0 0 -height 0 height] # (for a left to right, top to bottom image ) # first let's map the user coordinates start at offset x1,y1 on page self.code.extend([ 'gsave', '%s %s translate' % (x1, -y1 - drawheight), # need to start are lower left of image '%s %s scale' % (drawwidth, drawheight), '/scanline %d 3 mul string def' % imgwidth # scanline by multiples of image width ]) # now push the dimensions and depth info onto the stack # and push the ImageMatrix to map the source to the target rectangle (see above) # finally specify source (PS2 pp. 225 ) and by exmample self.code.extend([ '%s %s %s' % (imgwidth, imgheight, component_depth), '[%s %s %s %s %s %s]' % (imgwidth, 0, 0, -imgheight, 0, imgheight), '{ currentfile scanline readhexstring pop } false 3', 'colorimage ' ]) # data source output--now we just need to deliver a hex encode # series of lines of the right overall size can follow # piddlePDF again rawimage = getattr(myimage, 'tobytes', myimage.tostring)() assert len(rawimage) == imgwidth * imgheight, 'Wrong amount of data for image' #compressed = zlib.compress(rawimage) # no zlib at moment hex_encoded = self._AsciiHexEncode(rawimage) # write in blocks of 78 chars per line outstream = StringIO(hex_encoded) dataline = outstream.read(78) while dataline != "": self.code.append(dataline) dataline = outstream.read(78) self.code.append('% end of image data') # for clarity self.code.append('grestore') # return coordinates to normal