def parse_stream(self, data):
hdr = header.parse_stream(data)
if hdr.signature[0] == "C":
import zlib
data.seek(0)
data = StringIO(data.read(8) + zlib.decompress(data.read()))
return Struct.parse_stream(self, data)
def add_packet(self, new_packet): """ Adds the packet to the unfinished object. Updates all things appropriately :param string newPacket: The incoming packet """ packet = unpack(new_packet) reader = StringIO(packet[1]) # Get the actual packet data as a StringIO packet_number = _bin_unpack(reader.read(4)) # Read the 4-byte number of the packet # Try to write the info to the correct place self.info_buffer.seek(_UnfinishedPacket.MAX_PACKET_SIZE * packet_number) self.info_buffer.write(reader.read()) reader.close() self.info_buffer.seek(0) # Update incoming packet tracking data if(packet_number > self.last_received): self.last_received = packet_number for num in range(self.last_received,packet_number): self.missing_packets.append(num) else: self.missing_packets.remove(packet_number) # See if all packets are received. if(self.last_received == self.num_packets-1 and not self.missing_packets): self.done = True
class TBufferedTransport(TTransportBase,CReadableTransport):
"""Class that wraps another transport and buffers its I/O.
The implementation uses a (configurable) fixed-size read buffer
but buffers all writes until a flush is performed.
"""
DEFAULT_BUFFER = 4096
def __init__(self, trans, rbuf_size = DEFAULT_BUFFER):
self.__trans = trans
self.__wbuf = StringIO()
self.__rbuf = StringIO("")
self.__rbuf_size = rbuf_size
def isOpen(self):
return self.__trans.isOpen()
def open(self):
return self.__trans.open()
def close(self):
return self.__trans.close()
def read(self, sz):
ret = self.__rbuf.read(sz)
if len(ret) != 0:
return ret
self.__rbuf = StringIO(self.__trans.read(max(sz, self.__rbuf_size)))
return self.__rbuf.read(sz)
def write(self, buf):
self.__wbuf.write(buf)
def flush(self):
out = self.__wbuf.getvalue()
# reset wbuf before write/flush to preserve state on underlying failure
self.__wbuf = StringIO()
self.__trans.write(out)
self.__trans.flush()
# Implement the CReadableTransport interface.
@property
def cstringio_buf(self):
return self.__rbuf
def cstringio_refill(self, partialread, reqlen):
retstring = partialread
if reqlen < self.__rbuf_size:
# try to make a read of as much as we can.
retstring += self.__trans.read(self.__rbuf_size)
# but make sure we do read reqlen bytes.
if len(retstring) < reqlen:
retstring += self.__trans.readAll(reqlen - len(retstring))
self.__rbuf = StringIO(retstring)
return self.__rbuf
def cacheImageFile(filename):
"Processes the image as if for encoding, saves to a file ending in AHX"
from PIL import Image
import zlib
img1 = Image.open(filename)
img = img1.convert('RGB')
imgwidth, imgheight = img.size
code = []
code.append('BI') # begin image
# this describes what is in the image itself
code.append('/W %s /H %s /BPC 8 /CS /RGB /F [/A85 /Fl]' % (imgwidth, imgheight))
code.append('ID')
#use a flate filter and Ascii Base 85
raw = getattr(img, 'tobytes', img.tostring)()
assert len(raw) == imgwidth * imgheight, "Wrong amount of data for image"
compressed = zlib.compress(raw) #this bit is very fast...
encoded = _AsciiBase85Encode(compressed) #...sadly this isn't
#write in blocks of 60 characters per line
outstream = StringIO(encoded)
dataline = outstream.read(60)
while dataline != "":
code.append(dataline)
dataline = outstream.read(60)
code.append('EI')
#save it to a file
cachedname = os.path.splitext(filename)[0] + '.a85'
f = open(cachedname, 'wb')
f.write(string.join(code, LINEEND) + LINEEND)
f.close()
print('cached image as %s' % cachedname)
def run_test(conf_url=querymod.configuration.get('url', None)):
""" Run the unit tests and make sure the server is online."""
if conf_url is None:
raise ValueError("Please create a local api_config.json file in the "
"root directory of the repository with 'url' defined "
"with the root URL of the server. (No /rest or "
"/jsonrpc should be present.) Or provide URL on "
"command line.")
# Tell the test framework where to query
global url
url = conf_url
results = StringIO()
# Run the test
demo_test = unittest.TestLoader().loadTestsFromTestCase(TestAPI)
unittest.TextTestRunner(stream=results).run(demo_test)
# See if the end of the results says it passed
results.seek(results.tell() - 3)
if results.read() == "OK\n":
sys.exit(0)
else:
results.seek(0)
print(results.read())
sys.exit(1)
def filename_to_docname(filename):
"""
Convert the filename of a saved document to a documentname. Filenames have the
form "cnl-[documentname].save" where [documentname] is the name of the
document with escaped special characters
Positional arguments:
filename -- Name of the file with escaped special characters
Return value: Name of the document without escaped special characters.
"""
result = ""
input = StringIO(filename[4:-5])
while True:
char = input.read(1)
if char == "":
break
elif char == ':':
charcode = ""
while len(charcode) == 0 or charcode[-1] != ';':
charcode += input.read(1)
char = chr(int(charcode[:-1], 16))
result += char
return result
def _open(self):
header = StringIO(self.fp.read(24))
magic = header.read(4)
if magic != "FTEX":
raise ValueError("not a FTEX file")
version = unpack("i", header.read(4))
self.size = unpack("ii", header.read(8))
linesize = (self.size[0] + 3) / 4 * 8
mipmap_count, format_count = unpack("ii", header.read(8))
self.mode = "RGB"
self.tile = []
for i in range(format_count):
format, where = unpack("ii", self.fp.read(8))
if format == 0:
data = []
self.fp.seek(where)
size, = unpack("i", self.fp.read(4))
for yb in xrange((self.size[1] + 3) / 4):
decoded = dxtc.decodeDXT1(self.fp.read(linesize))
for d in decoded:
data.append(d)
data = "".join(data[:self.size[1]])
self.im = Image.core.new(self.mode, self.size)
return self.fromstring(data)
elif format == 1: # Uncompressed RGB
self.tile.append(("raw", (0, 0) + self.size, where+4, (self.mode, 0, 1)))
else:
raise ValueError("Invalid texture format (expected 0 or 1, got %i)" % (format))
class TFramedTransport(TTransportBase, CReadableTransport):
"""Class that wraps another transport and frames its I/O when writing."""
def __init__(self, trans,):
self.__trans = trans
self.__rbuf = StringIO()
self.__wbuf = StringIO()
def isOpen(self):
return self.__trans.isOpen()
def open(self):
return self.__trans.open()
def close(self):
return self.__trans.close()
def read(self, sz):
ret = self.__rbuf.read(sz)
if len(ret) != 0:
return ret
self.readFrame()
return self.__rbuf.read(sz)
def readFrame(self):
buff = self.__trans.readAll(4)
sz, = unpack('!i', buff)
self.__rbuf = StringIO(self.__trans.readAll(sz))
def write(self, buf):
self.__wbuf.write(buf)
def flush(self):
wout = self.__wbuf.getvalue()
wsz = len(wout)
# reset wbuf before write/flush to preserve state on underlying failure
self.__wbuf = StringIO()
# N.B.: Doing this string concatenation is WAY cheaper than making
# two separate calls to the underlying socket object. Socket writes in
# Python turn out to be REALLY expensive, but it seems to do a pretty
# good job of managing string buffer operations without excessive copies
buf = pack("!i", wsz) + wout
self.__trans.write(buf)
self.__trans.flush()
# Implement the CReadableTransport interface.
@property
def cstringio_buf(self):
return self.__rbuf
def cstringio_refill(self, prefix, reqlen):
# self.__rbuf will already be empty here because fastbinary doesn't
# ask for a refill until the previous buffer is empty. Therefore,
# we can start reading new frames immediately.
while len(prefix) < reqlen:
self.readFrame()
prefix += self.__rbuf.getvalue()
self.__rbuf = StringIO(prefix)
return self.__rbuf
def test_magic_autopx_nonblocking(self):
ip = get_ipython()
v = self.client[-1]
v.activate()
v.block=False
sio = StringIO()
savestdout = sys.stdout
sys.stdout = sio
ip.magic_autopx()
ip.run_cell('\n'.join(('a=5','b=10','c=0')))
ip.run_cell('print b')
ip.run_cell("b/c")
ip.run_code(compile('b*=2', '', 'single'))
ip.magic_autopx()
sys.stdout = savestdout
sio.read()
output = sio.buf.strip()
self.assertTrue(output.startswith('%autopx enabled'))
self.assertTrue(output.endswith('%autopx disabled'))
self.assertFalse('ZeroDivisionError' in output)
ar = v.get_result(-2)
self.assertEquals(v['a'], 5)
self.assertEquals(v['b'], 20)
self.assertRaisesRemote(ZeroDivisionError, ar.get)
class FifoBuffer(io.TextIOBase):
def __init__(self):
self.buf = StringIO()
self.len = 0
def read(self):
"""Reads data from buffer"""
self.buf.seek(0)
l = self.len
self.len=0
res = self.buf.read(l)
self.buf.seek(0)
return res
def write(self, arg):
self.len = self.len + self.buf.write(arg)
def peek(self):
x = self.buf.tell()
self.buf.seek(0)
res = self.buf.read(x)
self.buf.seek(x)
return res
def normalize_cmd():
parser = ArgumentParser()
parser.add_argument(
'-C',
'--check',
dest='check',
action='store_true',
default=False,
help=('Do not modify buildout file, '
'only return if file would be modified'))
parser.add_argument('configfiles',
nargs='*',
help=('The configfile to normalize in place, '
'or "-" to read the config file from stdin '
'and return the result to stdout'))
args = parser.parse_args()
if not len(args.configfiles):
logger.exception('You must choose config files to normalize ',
'I won\'t guess them for you!')
sys.exit(3)
outstreams = {}
for configfile in args.configfiles:
if len(args.configfiles) == 1 and configfile == '-':
instream = StringIO()
instream.write(sys.stdin.read())
instream.seek(0)
pipe = True
else:
instream = open(configfile, encoding='utf-8')
pipe = False
outstream = StringIO()
outstreams[configfile] = outstream
try:
sort(instream, outstream)
except Exception:
logger.exception('Could not parse file %r', configfile)
return sys.exit(3)
else:
instream.seek(0)
outstream.seek(0)
changed = instream.read() != outstream.read()
outstream.seek(0)
if not changed:
if pipe:
sys.stdout.write(outstream.read())
sys.exit(0)
else:
if args.check:
logger.error('File is not normalized')
sys.exit(3)
else:
if pipe:
sys.stdout.write(outstream.read())
if not pipe:
for outfile, outstream in outstreams.items():
open(outfile, 'w', encoding='utf-8').write(outstream.read())
def __decode_blp2(self):
header = StringIO(self.fp.read(20 + 16 * 4 + 16 * 4))
magic, compression = unpack(b"<4si", header.read(8))
encoding, alphaDepth, alphaEncoding, hasMips = unpack(b"<4b", header.read(4))
self.size = unpack(b"<II", header.read(8))
offsets = unpack(b"<16I", header.read(16 * 4))
lengths = unpack(b"<16I", header.read(16 * 4))
palette_data = self.fp.read(256 * 4)
self.mode = "RGB"
self.tile = []
if compression == 1: # Uncompressed or DirectX compression
data = []
self.fp.seek(offsets[0])
if encoding == 1: # uncompressed
palette = getpalette(palette_data)
_data = StringIO(self.fp.read(lengths[0]))
while True:
try:
offset, = unpack(b"<B", _data.read(1))
except StructError:
break
b, g, r, a = palette[offset]
data.append(pack(b"<BBB", r, g, b))
elif encoding == 2: # directx compression
self.mode = "RGBA" if alphaDepth else "RGB"
if alphaEncoding == 0: # DXT1
linesize = (self.size[0] + 3) / 4 * 8
for yb in xrange((self.size[1] + 3) / 4):
decoded = dxtc.decodeDXT1(self.fp.read(linesize), alpha=alphaDepth)
for d in decoded:
data.append(d)
elif alphaEncoding == 1: # DXT3
linesize = (self.size[0] + 3) / 4 * 16
for yb in xrange((self.size[1] + 3) / 4):
decoded = dxtc.decodeDXT3(self.fp.read(linesize))
for d in decoded:
data.append(d)
elif alphaEncoding == 7: # DXT5
linesize = (self.size[0] + 3) / 4 * 16
for yb in xrange((self.size[1] + 3) / 4):
decoded = dxtc.decodeDXT5(self.fp.read(linesize))
for d in decoded:
data.append(d)
else:
raise ValueError("Expected alphaEncoding 0, 1 or 7, got %i instead" % (alphaEncoding))
self.im = Image.core.new(self.mode, self.size)
data = "".join(data)
self.fromstring(data)
def grade_string(expected, actual):
"""Grades a single response against the true (actual) output.
Possible return values are:
* MATCH_EXACT (perfect match)
* MATCH_LINES (correct number of lines, non-formatting characters match)
* MATCH_VALUES (non-formatting characters match)
* MATCH_NONE (no match)"""
# Convert to universal newlines, strip extraneous whitespace
expected_io = StringIO(unicode(expected.strip()), newline=None)
actual_io = StringIO(unicode(actual.strip()), newline=None)
expected_str = expected_io.read()
actual_str = actual_io.read()
# Pefect match
if expected_str == actual_str:
return MATCH_EXACT
format_chars = ['[', ']', ',', ' ', '\n', '"', '\'']
table = dict.fromkeys(map(ord, format_chars), None)
expected_io.seek(0)
expected_lines = [line.strip() for line in expected_io.readlines()]
actual_io.seek(0)
actual_lines = [line.strip() for line in actual_io.readlines()]
# Remove blank lines
removed_blanks = False
if len(expected_lines) != len(actual_lines):
actual_lines = [line for line in actual_lines if len(line.strip()) > 0]
removed_blanks = True
# Check for line by line exact/partial match
if len(expected_lines) == len(actual_lines):
exact_match = True
partial_match = False
for (e_line, a_line) in zip(expected_lines, actual_lines):
if e_line != a_line:
exact_match = False
if (e_line.translate(table).lower() == a_line.translate(table).lower()):
partial_match = True
else:
partial_match = False
break
if exact_match:
return MATCH_EXACT if not removed_blanks else MATCH_LINES
elif partial_match:
return MATCH_LINES
# Check for partial match of values only
if expected_str.translate(table).lower() == actual_str.translate(table).lower():
return MATCH_VALUES
return MATCH_NONE
def xpm2png(value):
xpm = '''/* XPM */
static char * ala_xpm[] = {
"%s"};''' % '",\n"'.join(value)
xpm = xpm.replace('None', '#ffffff').replace('Black', '#000000')
io = StringIO(xpm)
im = Image.open(io)
outio = StringIO()
im.save(outio, 'png')
# I don't understand why, but this is needed to get data to buf
outio.read()
return outio.buf.encode('base64')
def tree_from_str(data):
sio = StringIO(data)
magic_str = sio.read(4)
if magic_str != b'PFTR':
raise RuntimeError("magic string not found")
version_str = sio.read(4)
version = struct.unpack("!I", version_str)[0]
if version != TREE_VER_P1:
raise RuntimeError("tree version not supported")
return pickle.load(sio)
def toarray(string, wordsize=8):
# Normalize first
string = string.strip().replace(" ", "")
pad_len = wordsize - (len(string) % wordsize)
if pad_len < wordsize:
string = ("0" * pad_len) + string
stream = StringIO(string)
chunks = []
chunk = stream.read(wordsize)
while chunk:
chunks.append(chunk)
chunk = stream.read(wordsize)
return ', '.join('0x{}'.format(x.lower()) for x in reversed(chunks))
def feed(self, markup):
if isinstance(markup, str):
markup = StringIO(markup)
# Call feed() at least once, even if the markup is empty,
# or the parser won't be initialized.
data = markup.read(self.CHUNK_SIZE)
self.parser.feed(data)
while data != '':
# Now call feed() on the rest of the data, chunk by chunk.
data = markup.read(self.CHUNK_SIZE)
if data != '':
self.parser.feed(data)
self.parser.close()
def grade_string(expected, actual):
# Convert to universal newlines, strip extraneous whitespace
expected_io = StringIO(unicode(expected.strip()), newline=None)
actual_io = StringIO(unicode(actual.strip()), newline=None)
expected_str = expected_io.read()
actual_str = actual_io.read()
# Pefect match
if expected_str == actual_str:
return "exact"
table = dict.fromkeys(map(ord, FORMAT_CHARS), None)
expected_io.seek(0)
expected_lines = [line.strip() for line in expected_io.readlines()]
actual_io.seek(0)
actual_lines = [line.strip() for line in actual_io.readlines()]
# Remove blank lines
removed_blanks = False
if len(expected_lines) != len(actual_lines):
actual_lines = [line for line in actual_lines if len(line.strip()) > 0]
removed_blanks = True
# Check for line by line exact/partial match
if len(expected_lines) == len(actual_lines):
exact_match = True
partial_match = False
for (e_line, a_line) in zip(expected_lines, actual_lines):
if e_line != a_line:
exact_match = False
if (e_line.translate(table).lower() == a_line.translate(table).lower()):
partial_match = True
else:
partial_match = False
break
if exact_match:
return "exact" if not removed_blanks else "line"
elif partial_match:
return "line"
# Check for partial match of values only
if expected_str.translate(table).lower() == actual_str.translate(table).lower():
return "values"
return None
def RunLengthDecode(data):
f = StringIO(data)
decompressed = ''
runLength = ord(f.read(1))
while runLength:
if runLength < 128:
decompressed += f.read(runLength + 1)
if runLength > 128:
decompressed += f.read(1) * (257 - runLength)
if runLength == 128:
break
runLength = ord(f.read(1))
# return sub(r'(\d+)(\D)', lambda m: m.group(2) * int(m.group(1)), data)
return decompressed
def parse(data, key):
fp = BytesIO(data)
build_dictionary(fp, key)
records = Record.parse(fp)
out = StringIO()
print_records(records, fp=out)
out.seek(0)
if pygments is not None:
print(pygments.highlight(out.read(),
pygments.lexers.get_lexer_by_name('XML'),
pygments.formatters.get_formatter_by_name('terminal')))
else:
print(out.read())
def bsdiffParse(self): diff = StringIO(self.rleUnpack()) ctrlBlockSize, diffBlockSize, sizeAfter = self.__bsdiffParseHeader(diff) ## # The data part of the file is divded into three parts # * The control block, which contains the control part every chunk # * The diff block, which contains the diff chunks # * And the extra block, which contains the extra chunks ctrlBlock = StringIO(diff.read(ctrlBlockSize)) diffBlock = StringIO(diff.read(diffBlockSize)) extraBlock = StringIO(diff.read()) diff.close() return ctrlBlock, diffBlock, extraBlock
def test_print_items(self, bold, PL, PD):
from kamaki.cli.utils import print_items, INDENT_TAB
for args in product(
(
42, None, 'simple outputs',
[1, 2, 3], {1: 1, 2: 2}, (3, 4),
({'k': 1, 'id': 2}, [5, 6, 7], (8, 9), '10')),
(('id', 'name'), ('something', 2), ('lala', )),
(False, True),
(False, True)):
items, title, with_enumeration, with_redundancy = args
pl_counter, pd_counter = len(PL.mock_calls), len(PD.mock_calls)
bold_counter, out_counter = len(bold.mock_calls), 0
out = StringIO()
print_items(*args, out=out)
out.seek(0)
if not (isinstance(items, dict) or isinstance(
items, list) or isinstance(items, tuple)):
if items:
self.assertEqual(out.getvalue(), '%s\n' % items)
else:
for i, item in enumerate(items):
if with_enumeration:
exp_str = '%s. ' % (i + 1)
self.assertEqual(out.read(len(exp_str)), exp_str)
if isinstance(item, dict):
title = sorted(set(title).intersection(item))
pick = item.get if with_redundancy else item.pop
header = ' '.join('%s' % pick(key) for key in title)
if header:
self.assertEqual(
bold.mock_calls[bold_counter], call(header))
self.assertEqual(out.read(5), 'bold\n')
bold_counter += 1
else:
out.read(1)
self.assertEqual(
PD.mock_calls[pd_counter],
call(item, indent=INDENT_TAB, out=out))
pd_counter += 1
elif isinstance(item, list) or isinstance(item, tuple):
self.assertEqual(
PL.mock_calls[pl_counter],
call(item, indent=INDENT_TAB, out=out))
pl_counter += 1
else:
exp_str = u' %s\n' % item
self.assertEqual(out.read(len(exp_str)), exp_str)
def run_pip_main(cls, *args, **kwargs):
import pip
args = list(args)
check_output = kwargs.pop('check_output', False)
if check_output:
from io import StringIO
out = StringIO()
sys.stdout = out
try:
pip.main(args)
except:
traceback.print_exc()
finally:
sys.stdout = sys.__stdout__
out.seek(0)
pipdata = out.read()
out.close()
print(pipdata)
return pipdata
else:
return pip.main(args)
def test_versions_and_sources_last(self):
cfg = self.given_a_file_in_test_dir('buildout.cfg', '''\
[buildout]
[versions]
[sources]
[www]
[zzz]
[aaa]''')
output = StringIO()
sort(open(cfg), output)
output.seek(0)
expected = '''\
[buildout]
[aaa]
[www]
[zzz]
[sources]
[versions]
'''
self.assertEqual(expected, output.read())
def test_exclusion_regexp(self):
entries = [{'MESSAGE': 'exclude this'},
{'MESSAGE': 'message 1'},
{'MESSAGE': 'exclude that'},
{'MESSAGE': 'message 2'}]
(flexmock(journal.Reader)
.should_receive('get_next')
.and_return(entries[0])
.and_return(entries[1])
.and_return(entries[2])
.and_return(entries[3])
.and_return({}))
exclusions = [{'MESSAGE': ['/1/']}, # shouldn't exclude anything
{'MESSAGE': ['/exclude th/']},
{'MESSAGE': ['/exclude/']}]
formatter = EntryFormatter()
jfilter = JournalFilter(journal.Reader(), [formatter],
default_exclusions=exclusions)
output = StringIO()
jfilter.format(output)
output.seek(0)
lines = output.read().splitlines()
assert lines == [entry['MESSAGE']
for entry in [entries[1]] + [entries[3]]]
stats = jfilter.get_statistics()
for stat in stats:
if stat.exclusion['MESSAGE'] == ['/1/']:
assert stat.hits == 0
break
class LoggerTest(unittest.TestCase):
"""Testcase for the Logger actor.
"""
def setUp(self):
self._file_like = StringIO()
self._logger = Logger.start(file_like=self._file_like).proxy()
def tearDown(self):
self._logger.stop()
def _stop_logger(self):
self._logger.stop()
self._logger.actor_stopped.get().wait()
def test_records_are_recorded(self):
"""Verify that logging records produces messages.
"""
mutation_record = MutationRecord(
'foo', 'foo.py', 'operator',
{'description': 'desc',
'line_number': 3},
None)
test_result = TestResult(Outcome.KILLED, 'ok')
self._logger.handle_result(mutation_record, test_result)
self._stop_logger()
self._file_like.flush()
self._file_like.seek(0)
self.assertGreater(len(self._file_like.read()), 0)
def get_all_rows(es_params, request):
buffer_ = StringIO()
writer = csv.writer(buffer_)
writer.writerow([feature for feature in es_params['features']])
ds = Datasets().activate_dataset(request.session)
es_m = ds.build_manager(ES_Manager)
es_m.build(es_params)
es_m.set_query_parameter('size', ES_SCROLL_BATCH)
features = sorted(es_params['features'])
response = es_m.scroll()
scroll_id = response['_scroll_id']
hits = response['hits']['hits']
while hits:
process_hits(hits, features, write=True, writer=writer)
buffer_.seek(0)
data = buffer_.read()
buffer_.seek(0)
buffer_.truncate()
yield data
response = es_m.scroll(scroll_id=scroll_id)
hits = response['hits']['hits']
scroll_id = response['_scroll_id']
def save(self, force_update=False, force_insert=False,
thumb_size=(IMAGE_MAX_HEIGHT, IMAGE_MAX_WIDTH)):
"""Save the article.
This will save thumbnail on disk and then save the model in database.
"""
self.slug = slugify(self.title)
if has_changed(self, 'image') and self.image:
# TODO : delete old image
image = Image.open(self.image)
if image.mode not in ('L', 'RGB'):
image = image.convert('RGB')
image.thumbnail(thumb_size, Image.ANTIALIAS)
# save the thumbnail to memory
temp_handle = StringIO()
image.save(temp_handle, 'png')
temp_handle.seek(0) # rewind the file
# save to the thumbnail field
suf = SimpleUploadedFile(os.path.split(self.image.name)[-1],
temp_handle.read(),
content_type='image/png')
self.thumbnail.save('{}.png'.format(suf.name), suf, save=False)
# save the image object
super().save(force_update, force_insert)
else:
super().save()
def test_flush_pipes_data_between_streams(self):
a = StringIO(u'food')
b = StringIO()
pump = io.Pump(a, b)
pump.flush(3)
expect(a.read(1)).to.equal('d')
expect(b.getvalue()).to.equal('foo')
def render_me(element):
''' a utility to render an elment so you can see if its doing the right thing'''
f = StringIO()
element.render(f)
f.seek(0)
return f.read()
class CORATransformer:
"""
This class captures our understanding of the agreed format
of the UKIS survey.
"""
_MAP_YN_10 = {
'yes': '1',
'no': '0',
}
_MAP_YN_01 = {
'yes': '0',
'no': '1',
}
_MAP_YN_21 = {
'yes': '10',
'no': '01',
}
_MAP_IMPORTANCE = {
'not important': '0001',
'low': '0010',
'medium': '0100',
'high': '1000',
}
_MAP_PROPORTIONS = {
'none': '0001',
'less than 40%': '0010',
'40-90%': '0100',
'over 90%': '1000',
}
class _Format(enum.Enum):
zeroone = re.compile("^[01]{1}$")
onetwo = re.compile("^[12]{1}$")
twobin = re.compile("^[0-1]{1,2}$")
twodigits = re.compile("^[0-9]{1,2}$")
threedigits = re.compile("^[0-9]{1,3}$")
sixdigits = re.compile("^[0-9]{1,6}$")
sevendigits = re.compile("^[0-9]{1,7}$")
onehotfour = re.compile("^(1000|0100|0010|0001|0000)$")
yesno = re.compile("^yes|no$")
yesnodk = re.compile("^yes|no|(don.+t know)$")
class _Processor:
@staticmethod
def false(q, d):
return "0"
@staticmethod
def checkbox(q, d):
return '0' if q not in d else '1'
@staticmethod
def checkboxtwobit(q, d):
return '00' if q not in d else '10'
@staticmethod
def radioyn10(q, d):
return '0' if q not in d else CORATransformer._MAP_YN_10.get(
d[q].lower(), '0')
@staticmethod
def radioyn01(q, d):
return '1' if q not in d else CORATransformer._MAP_YN_01.get(
d[q].lower(), '1')
@staticmethod
def radioyn21(q, d):
return '00' if q not in d else CORATransformer._MAP_YN_21.get(
d[q].lower(), '00')
@staticmethod
def radioyndk(q, d):
return '1' if d.get(q, "").lower() == "yes" else '0'
@staticmethod
def radioimportance(q, d):
return '0000' if q not in d else CORATransformer._MAP_IMPORTANCE.get(
d[q].lower(), '0000')
@staticmethod
def radioproportion(q, d):
return '0000' if q not in d else CORATransformer._MAP_PROPORTIONS.get(
d[q].lower(), '0000')
@staticmethod
def zeropadthree(q, d):
return '' if q not in d else '{0:03d}'.format(int(d[q]))
@staticmethod
def zeropadtwo(q, d):
return '' if q not in d else '{0:02d}'.format(int(d[q]))
@staticmethod
def dividebythousand(q, d):
if q not in d:
return ''
if d[q].isdigit():
return str(int(d[q]) // 1000)
return ''
@staticmethod
def numbertype(q, d):
return '' if q not in d else d[q]
@staticmethod
def comment(q, d):
return '0' if q not in d else '1' if len(d[q].strip()) > 0 else '0'
_defn = [
(range(1, 4, 1), "0", _Format.zeroone, _Processor.false),
(range(210, 250, 10), "0", _Format.zeroone, _Processor.checkbox),
(range(410, 440, 10), "0", _Format.zeroone, _Processor.radioyn10),
(range(2310, 2350, 10), "0", _Format.zeroone, _Processor.radioyn10),
(range(1310, 1311, 1), "0", _Format.zeroone, _Processor.radioyn10),
(range(2675, 2678, 1), "0", _Format.zeroone, _Processor.checkbox),
(range(1410, 1411,
1), "", _Format.sixdigits, _Processor.dividebythousand),
(range(1320, 1321, 1), "0", _Format.zeroone, _Processor.radioyn10),
(range(1420, 1421,
1), "", _Format.sixdigits, _Processor.dividebythousand),
(range(1331, 1334, 1), "0", _Format.zeroone, _Processor.checkbox),
(range(1430, 1431,
1), "", _Format.sixdigits, _Processor.dividebythousand),
(range(1340, 1341, 1), "0", _Format.zeroone, _Processor.radioyn10),
(range(1440, 1441,
1), "", _Format.sixdigits, _Processor.dividebythousand),
(range(1350, 1351, 1), "0", _Format.zeroone, _Processor.radioyn10),
(range(1450, 1451,
1), "", _Format.sixdigits, _Processor.dividebythousand),
(range(1360, 1361, 1), "0", _Format.zeroone, _Processor.radioyn10),
(range(1460, 1461,
1), "", _Format.sixdigits, _Processor.dividebythousand),
(range(1371, 1375, 1), "0", _Format.zeroone, _Processor.checkbox),
(range(1470, 1471,
1), "", _Format.sixdigits, _Processor.dividebythousand),
(range(510, 511, 1), "00", _Format.twobin, _Processor.radioyn21),
(range(610, 640, 10), "0", _Format.zeroone, _Processor.checkbox),
(range(520, 521, 1), "00", _Format.twobin, _Processor.radioyn21),
(range(601, 604, 1), "0", _Format.zeroone, _Processor.checkbox),
(range(710, 730, 10), "0", _Format.zeroone, _Processor.radioyn10),
(range(810, 850,
10), "", _Format.threedigits, _Processor.zeropadthree),
(range(900, 901, 1), "00", _Format.twobin, _Processor.radioyn21),
(range(1010, 1040, 10), "0", _Format.zeroone, _Processor.checkbox),
(range(1100, 1101, 1), "00", _Format.twobin, _Processor.radioyn21),
(range(1510, 1540, 10), "0", _Format.zeroone, _Processor.radioyn10),
(range(2657, 2668,
1), "0000", _Format.onehotfour, _Processor.radioimportance),
(range(2011, 2012, 1), "0", _Format.zeroone, _Processor.checkbox),
(range(2020, 2050, 10), "0", _Format.zeroone, _Processor.checkbox),
(range(1210, 1212,
1), "0000", _Format.onehotfour, _Processor.radioimportance),
(range(1220, 1300,
10), "0000", _Format.onehotfour, _Processor.radioimportance),
(range(1212, 1214,
1), "0000", _Format.onehotfour, _Processor.radioimportance),
(range(1601, 1602,
1), "0000", _Format.onehotfour, _Processor.radioimportance),
(range(1620, 1621,
1), "0000", _Format.onehotfour, _Processor.radioimportance),
(range(1610, 1612,
1), "0000", _Format.onehotfour, _Processor.radioimportance),
(range(1631, 1633,
1), "0000", _Format.onehotfour, _Processor.radioimportance),
(range(1640, 1700,
10), "0000", _Format.onehotfour, _Processor.radioimportance),
(range(1811, 1815,
1), "00", _Format.twobin, _Processor.checkboxtwobit),
(range(1821, 1825,
1), "00", _Format.twobin, _Processor.checkboxtwobit),
(range(1881, 1885,
1), "00", _Format.twobin, _Processor.checkboxtwobit),
(range(1891, 1895,
1), "00", _Format.twobin, _Processor.checkboxtwobit),
(range(1841, 1845,
1), "00", _Format.twobin, _Processor.checkboxtwobit),
(range(1851, 1855,
1), "00", _Format.twobin, _Processor.checkboxtwobit),
(range(1861, 1865,
1), "00", _Format.twobin, _Processor.checkboxtwobit),
(range(1871, 1875,
1), "00", _Format.twobin, _Processor.checkboxtwobit),
(range(2650, 2657,
1), "0000", _Format.onehotfour, _Processor.radioproportion),
(range(2668, 2671, 1), "0", _Format.zeroone, _Processor.radioyn10),
(range(2672, 2674, 1), "0", _Format.zeroone, _Processor.radioyndk),
(range(2410, 2430,
10), "", _Format.sixdigits, _Processor.dividebythousand),
(range(2440, 2450,
10), "", _Format.sixdigits, _Processor.dividebythousand),
(range(2510, 2530,
10), "", _Format.sevendigits, _Processor.numbertype),
(range(2610, 2630,
10), "", _Format.threedigits, _Processor.zeropadthree),
(range(2631, 2637, 1), "0", _Format.zeroone, _Processor.checkbox),
(range(2678, 2679,
1), "0000", _Format.onehotfour, _Processor.radioimportance),
(range(2700, 2701, 1), "0", _Format.zeroone, _Processor.comment),
(range(2801, 2802, 1), "", _Format.threedigits, _Processor.numbertype),
(range(2800, 2801, 1), "", _Format.twodigits, _Processor.zeropadtwo),
(range(2900, 2901, 1), "00", _Format.twobin, _Processor.radioyn21),
]
def __init__(self, logger, survey, response_data, sequence_no=1000):
self._logger = logger
self._survey = survey
self._response = response_data
self._sequence_no = sequence_no
self._idbr = StringIO()
self._response_json = StringIO()
self._tkn = StringIO()
self.image_transformer = ImageTransformer(
self._logger,
self._survey,
self._response,
CoraPdfTransformerStyle(),
sequence_no=self._sequence_no,
base_image_path=SDX_FTP_IMAGE_PATH)
self._setup_logger()
def create_zip(self):
""" Create a in memory zip from a renumbered sequence"""
idbr_name = self._create_idbr()
tkn_name = self._create_tkn()
response_io_name = self._create_response_json()
self.image_transformer.zip.append(
os.path.join(SDX_FTP_DATA_PATH, tkn_name), self._tkn.read())
self.image_transformer.zip.append(
os.path.join(SDX_FTP_RECEIPT_PATH, idbr_name), self._idbr.read())
self.image_transformer.get_zipped_images()
self.image_transformer.zip.append(
os.path.join(SDX_RESPONSE_JSON_PATH, response_io_name),
self._response_json.read())
self.image_transformer.zip.rewind()
def get_zip(self):
"""Get access to the in memory zip """
self.image_transformer.zip.rewind()
return self.image_transformer.zip.in_memory_zip
def _create_tkn(self):
data = CORATransformer._transform(self._response["data"])
output = CORATransformer._tkn_lines(
surveyCode=self._response["survey_id"],
ruRef=self._response["metadata"]["ru_ref"][:11],
period=self._response["collection"]["period"],
data=data)
for row in output:
self._tkn.write(row)
self._tkn.write("\n")
self._tkn.seek(0)
tkn_name = "{0}_{1:04}".format(self._survey["survey_id"],
self._sequence_no)
return tkn_name
def _create_idbr(self):
template = env.get_template('idbr.tmpl')
template_output = template.render(response=self._response)
submission_date = dateutil.parser.parse(self._response['submitted_at'])
submission_date_str = submission_date.strftime("%d%m")
# Format is RECddMM_batchId.DAT
# e.g. REC1001_30000.DAT for 10th January, batch 30000
idbr_name = "REC%s_%04d.DAT" % (submission_date_str, self._sequence_no)
self._idbr.write(template_output)
self._idbr.seek(0)
return idbr_name
def _setup_logger(self):
if self._survey:
if 'metadata' in self._survey:
metadata = self._survey['metadata']
self._logger = self._logger.bind(user_id=metadata['user_id'],
ru_ref=metadata['ru_ref'])
if 'tx_id' in self._survey:
self.tx_id = self._survey['tx_id']
self._logger = self._logger.bind(tx_id=self.tx_id)
def _create_response_json(self):
original_json_name = "%s_%04d.json" % (self._survey['survey_id'],
self._sequence_no)
self._response_json.write(json.dumps(self._response))
self._response_json.seek(0)
return original_json_name
@staticmethod
def _checks():
"""
Returns a dictionary mapping question ids to field formats.
"""
return OrderedDict([("{0:04}".format(i), check)
for rng, val, check, op in CORATransformer._defn
for i in rng])
@staticmethod
def _ops():
"""
Returns a dictionary mapping question ids to field operations.
"""
return OrderedDict([("{0:04}".format(i), op)
for rng, val, check, op in CORATransformer._defn
for i in rng])
@staticmethod
def _defaults():
"""
Returns a dictionary mapping question ids to default values.
"""
return OrderedDict([("{0:04}".format(i), val)
for rng, val, check, op in CORATransformer._defn
for i in rng])
@staticmethod
def _transform(data):
rv = CORATransformer._defaults()
ops = CORATransformer._ops()
# Don't know generation
if any(
data.get(q, "").lower().endswith("t know")
for q in ("2672", "2673")):
rv["2674"] = "1"
else:
rv["2674"] = "0"
for q in rv:
if q == '10001':
del rv[q]
# Run the processors
try:
op = ops[q]
except KeyError:
continue
else:
rv[q] = op(q, data)
# None-of-the-above generation
if not any(rv.get(k) == "1" for k in ("0410", "0420", "0430")):
rv["0440"] = "1"
else:
rv["0440"] = "0"
if not any(rv.get(k) == "1" for k in ("2668", "2669", "2670")):
rv["2671"] = "1"
else:
rv["2671"] = "0"
return rv
@staticmethod
def _tkn_lines(surveyCode, ruRef, period, data):
pageId = "1"
questionInstance = "0"
return [
":".join(
(surveyCode, ruRef, pageId, period, questionInstance, q, a))
for q, a in data.items()
]
def test_dump(json, nl):
buff = StringIO()
json.dump([1], buff, nl=nl)
buff.seek(0)
nl = "\n" if nl else ""
assert buff.read() == f"[1]{nl}"
TEST_FILE = pd.read_json(HERE / "FIXED_STATIONDATA.JSON")
# Prepare csv for regular "downloading" test
CSV_FILE = StringIO()
TEST_FILE.to_csv(CSV_FILE, sep=";")
CSV_FILE.seek(0)
@pytest.mark.xfail
@patch(
"wetterdienst.dwd.observations.fileindex.create_file_list_for_climate_observations",
MagicMock(return_value=[filename]),
)
@patch(
"wetterdienst.dwd.observations.access.download_climate_observations_data_parallel",
MagicMock(return_value=[(filename, BytesIO(CSV_FILE.read().encode()))]),
)
def test_collect_dwd_data_success():
""" Test for data collection """
"""
1. Scenario
This scenario makes sure we take fresh data and write it to the given folder, thus
we can run just another test afterwards as no old data is used
"""
assert collect_climate_observations_data(
station_ids=[1],
parameter=Parameter.CLIMATE_SUMMARY,
time_resolution=TimeResolution.DAILY,
period_type=PeriodType.HISTORICAL,
prefer_local=False,
write_file=True,
class Lexer(object):
"""
Simple Lexer base class. Provides basic lexer framework and
helper functionality (read/peek/pushback etc)
Each state is implemented using a method (lexXXXX) which should
match a single token and return a (token,lexYYYY) tuple, with lexYYYY
representing the next state. If token is None this is not emitted
and if lexYYYY is None or the lexer reaches the end of the
input stream the lexer exits.
The 'parse' method returns a generator that will return tokens
(the class also acts as an iterator)
The default start state is 'lexStart'
Input can either be a string/bytes or file object.
The approach is based loosely on Rob Pike's Go lexer presentation
(using generators rather than channels).
>>> p = Lexer("a bcd efgh")
>>> p.read()
'a'
>>> p.read()
' '
>>> p.peek(3)
'bcd'
>>> p.read(5)
'bcd e'
>>> p.pushback('e')
>>> p.read(4)
'efgh'
"""
escape_chars = '\\'
escape = {'n': '\n', 't': '\t', 'r': '\r'}
def __init__(self, f, debug=False):
if hasattr(f, 'read'):
self.f = f
elif type(f) == str:
self.f = StringIO(f)
elif type(f) == bytes:
self.f = StringIO(f.decode())
else:
raise ValueError("Invalid input")
self.debug = debug
self.q = collections.deque()
self.state = self.lexStart
self.escaped = False
self.eof = False
def __iter__(self):
return self.parse()
def next_token(self):
if self.debug:
print("STATE", self.state)
(tok, self.state) = self.state()
return tok
def parse(self):
while self.state is not None and not self.eof:
tok = self.next_token()
if tok:
yield tok
def read(self, n=1):
s = ""
while self.q and n > 0:
s += self.q.popleft()
n -= 1
s += self.f.read(n)
if s == '':
self.eof = True
if self.debug:
print("Read: >%s<" % repr(s))
return s
def peek(self, n=1):
s = ""
i = 0
while len(self.q) > i and n > 0:
s += self.q[i]
i += 1
n -= 1
r = self.f.read(n)
if n > 0 and r == '':
self.eof = True
self.q.extend(r)
if self.debug:
print("Peek : >%s<" % repr(s + r))
return s + r
def pushback(self, s):
p = collections.deque(s)
p.extend(self.q)
self.q = p
def readescaped(self):
c = self.read(1)
if c in self.escape_chars:
self.escaped = True
n = self.peek(3)
if n.isdigit():
n = self.read(3)
if self.debug:
print("Escape: >%s<" % n)
return chr(int(n, 8))
elif n[0] in 'x':
x = self.read(3)
if self.debug:
print("Escape: >%s<" % x)
return chr(int(x[1:], 16))
else:
c = self.read(1)
if self.debug:
print("Escape: >%s<" % c)
return self.escape.get(c, c)
else:
self.escaped = False
return c
def lexStart(self):
return (None, None)
# print(opcode, parameters)
if first_param_mode:
parameters[0] = data.index(
parameters[0])
if len(parameters) > 1:
if second_param_mode:
parameters[1] = data.index(
parameters[1])
if third_param_mode:
parameters[2] = data.index(
parameters[2])
# print(codes[opcode].__name__, parameters)
codes[opcode](data, *parameters, position)
# print(opcode)
position.set(position.get() +
offsets[opcode])
if opcode == 4:
break
opcode = data[position.get()]
if (opcode % 100) == 99:
break
if (opcode % 100) == 99:
break
else:
continue
break
print(values[0])
out.seek(0)
print(max(map(int, out.read().splitlines())))
class TFramedTransport(TTransportBase, CReadableTransport):
"""Class that wraps another transport and frames its I/O when writing."""
def __init__(self, trans,):
self.__trans = trans
self.__rbuf = StringIO()
self.__wbuf = StringIO()
def isOpen(self):
return self.__trans.isOpen()
def open(self):
return self.__trans.open()
def close(self):
return self.__trans.close()
def read(self, sz):
ret = self.__rbuf.read(sz)
if len(ret) != 0:
return ret
self.readFrame()
return self.__rbuf.read(sz)
def readFrame(self):
buff = self.__trans.readAll(4)
sz, = unpack('!i', buff)
self.__rbuf = StringIO(self.__trans.readAll(sz))
def write(self, buf):
self.__wbuf.write(buf)
def flush(self):
wout = self.__wbuf.getvalue()
wsz = len(wout)
# reset wbuf before write/flush to preserve state on underlying failure
self.__wbuf = StringIO()
# N.B.: Doing this string concatenation is WAY cheaper than making
# two separate calls to the underlying socket object. Socket writes in
# Python turn out to be REALLY expensive, but it seems to do a pretty
# good job of managing string buffer operations without excessive copies
buf = pack("!i", wsz) + wout
self.__trans.write(buf)
self.__trans.flush()
# Implement the CReadableTransport interface.
@property
def cstringio_buf(self):
return self.__rbuf
def cstringio_refill(self, prefix, reqlen):
# self.__rbuf will already be empty here because fastbinary doesn't
# ask for a refill until the previous buffer is empty. Therefore,
# we can start reading new frames immediately.
while len(prefix) < reqlen:
self.readFrame()
prefix += self.__rbuf.getvalue()
self.__rbuf = StringIO(prefix)
return self.__rbuf
class AsyncStringIO:
def __init__(self, string):
self.string_io = StringIO(string)
async def read(self, n=-1):
return self.string_io.read(n)
x = np.squeeze(v.values)
y = np.asarray(v.index, dtype=np.float64)
yi = np.interp(x, xp, yp)
abs_diff = np.abs(y - yi)
max_diff = np.max(abs_diff)
if max_diff > 0.05:
selected[np.where(abs_diff == max_diff)] = True
selected[v.index <= 10.0] = True
quantiles = list(np.squeeze(v[selected].index.values))
critical_values = list(np.squeeze(v[selected].values))
# Fix for first CV
critical_values[0] = 0.0
sio.write(k + " = (")
count = 0
for c, q in zip(critical_values, quantiles):
sio.write("(" + "{0:0.3f}".format(q) + ", " + "{0:0.4f}".format(c) +
")")
count += 1
if count % 4 == 0:
sio.write(",\n " + " " * len(k))
else:
sio.write(", ")
sio.write(")\n")
sio.write("kpss_critical_values['" + k + "'] = ")
sio.write("asarray(" + k + ")")
sio.write("\n")
sio.seek(0)
print(sio.read())
f = StringIO('Hello \n Wrold \n Goodbye') # 读内存数据
while True:
s = f.readline()
if s=='' :
break;
print(s.strip())
# StringIO操作的只能是str，如果要操作二进制数据，就需要使用BytesIO。
f = BytesIO()
f.write('中文'.encode('utf-8')) # 写二进制
print(f.getvalue())
#请注意，写入的不是str，而是经过UTF-8编码的bytes。和StringIO类似，可以用一个bytes初始化BytesIO，然后，像读文件一样读取：
f = BytesIO(b'\xe4\xb8\xad\xe6\x96\x87')
print(f.read())
''' ----------------------------------------------------------------------------------- part3 操作文件和目录'''
print(os.name)
print(os.uname())
print(os.environ) #环境变量
print(os.environ.get('PATH')) #获取某个环境变量的值
# 操作文件和目录，操作文件和目录的函数一部分放在os模块中，一部分放在os.path模块中
g = os.path.abspath('.') # 查看当前目录的绝对路径:
print(g)
#把两个路径合成一个时，不要直接拼字符串，而要通过os.path.join()函数，这样可以正确处理不同操作系统的路径分隔符/和\的问题
os.path.join('/home/wes/soft/pycharm/test', 'testdir') # 在某个目录下创建一个新目录，首先把新目录的完整路径表示出来
os.mkdir('/home/wes/soft/pycharm/test/testdir') # # 然后创建一个目录
os.rmdir('/home/wes/soft/pycharm/test/testdir') # 删除一个目录
class PresenterTests(TestCase):
def setUp(self) -> None:
self.result_output = StringIO()
self.error_output = StringIO()
self.renderer = TestingRepositoryRenderer()
self.presenter = PresenterImpl(
result_output=self.result_output,
error_output=self.error_output,
repository_renderer=self.renderer,
)
self.repo = PrefetchedRepository(
GithubRepository(owner="test", name="test"),
rev="test",
sha256="test",
options=PrefetchOptions(),
)
def test_write_to_error_output_when_presenting_prefetch_failure(
self) -> None:
self.presenter.present(
PrefetchFailure(
reason=PrefetchFailure.Reason.unable_to_calculate_sha256))
self.assertTrue(self.read_error_output())
def test_write_to_result_output_when_presenting_prefetched_repository(
self) -> None:
self.presenter.present(self.repo)
self.assertTrue(self.read_result_output())
def test_nothing_is_written_to_result_output_when_presenting_prefetch_failure(
self, ) -> None:
self.presenter.present(
PrefetchFailure(
reason=PrefetchFailure.Reason.unable_to_calculate_sha256))
self.assertFalse(self.read_result_output())
def test_nothing_is_written_to_error_output_when_presenting_prefetched_respository(
self, ) -> None:
self.presenter.present(self.repo)
self.assertFalse(self.read_error_output())
def test_rendered_repository_is_writted_to_result_output(self) -> None:
self.presenter.present(self.repo)
self.assertIn(
self.renderer.render_prefetched_repository(self.repo),
self.read_result_output(),
)
def test_that_exit_0_is_returned_when_repository_is_rendered(self) -> None:
return_code = self.presenter.present(self.repo)
self.assertEqual(return_code, 0)
def test_that_exit_1_is_returned_when_failure_is_rendered(self) -> None:
return_code = self.presenter.present(
PrefetchFailure(
reason=PrefetchFailure.Reason.unable_to_calculate_sha256))
self.assertEqual(return_code, 1)
def read_error_output(self) -> str:
self.error_output.seek(0)
return self.error_output.read()
def read_result_output(self) -> str:
self.result_output.seek(0)
return self.result_output.read()
def _drawImageLevel2(self, image, x1, y1, x2=None, y2=None): # Postscript Level2 version
try:
from PIL import Image
except ImportError:
print('Python Imaging Library not available')
return
# I don't have zlib -cwl
# try:
# import zlib
# except ImportError:
# print 'zlib not available'
# return
### what sort of image are we to draw
if image.mode == 'L':
print('found image.mode= L')
imBitsPerComponent = 8
imNumComponents = 1
myimage = image
elif image.mode == '1':
print('found image.mode= 1')
myimage = image.convert('L')
imNumComponents = 1
myimage = image
else:
myimage = image.convert('RGB')
imNumComponents = 3
imBitsPerComponent = 8
imwidth, imheight = myimage.size
# print 'imwidth = %s, imheight = %s' % myimage.size
if not x2:
x2 = imwidth + x1
if not y2:
y2 = y1 + imheight
drawwidth = x2 - x1
drawheight = y2 - y1
self.code.extend([
'gsave',
'%s %s translate' % (x1, -y1 - drawheight), # need to start are lower left of image
'%s %s scale' % (drawwidth, drawheight)
])
if imNumComponents == 3:
self.code.append('/DeviceRGB setcolorspace')
elif imNumComponents == 1:
self.code.append('/DeviceGray setcolorspace')
print('setting colorspace gray')
# create the image dictionary
self.code.append("""
<<
/ImageType 1
/Width %d /Height %d %% dimensions of source image
/BitsPerComponent %d""" % (imwidth, imheight, imBitsPerComponent))
if imNumComponents == 1:
self.code.append('/Decode [0 1]')
if imNumComponents == 3:
self.code.append('/Decode [0 1 0 1 0 1] %% decode color values normally')
self.code.extend(['/ImageMatrix [%s 0 0 %s 0 %s]' % (imwidth, -imheight, imheight),
'/DataSource currentfile /ASCIIHexDecode filter', '>> % End image dictionary',
'image'])
# after image operator just need to dump image dat to file as hexstring
rawimage = getattr(myimage, 'tobytes', myimage.tostring)()
assert len(rawimage) == imwidth * imheight, 'Wrong amount of data for image'
#compressed = zlib.compress(rawimage) # no zlib at moment
hex_encoded = self._AsciiHexEncode(rawimage)
# write in blocks of 78 chars per line
outstream = StringIO(hex_encoded)
dataline = outstream.read(78)
while dataline != "":
self.code.append(dataline)
dataline = outstream.read(78)
self.code.append('> % end of image data') # > is EOD for hex encoded filterfor clarity
self.code.append('grestore') # return coordinates to normal
def dumps(self,
reqs,
project: Optional[RootDependency] = None,
content: Optional[str] = None) -> str:
if content:
doc = yaml_load(content, safe=False)
else:
doc = dict()
doc['name'] = project.name
# channels
if 'channels' not in doc:
doc['channels'] = []
channels = set()
# get channels
for req in reqs:
if isinstance(req.dep.repo, CondaRepo):
channels.update(req.dep.repo.channels)
# remove old channels
for index, channel in reversed(list(enumerate(doc['channels']))):
if channel not in channels:
del doc['channels'][index]
# add new channels
for channel in channels:
if channel not in doc['channels']:
doc['channels'].append(channel)
# add default channel
if not doc['channels']:
doc['channels'] = ['defaults']
# dependencies
if 'dependencies' not in doc:
doc['dependencies'] = []
deps = {req.name: req.version for req in reqs}
# remove old deps
represented = set()
for index, dep in reversed(list(enumerate(doc['dependencies']))):
parsed = CondaRepo.parse_req(dep)
name = canonicalize_name(parsed['name'])
represented.add(name)
if name not in deps:
# remove
del doc['dependencies'][index]
elif parsed.get('version', '*') != deps[name]:
# update
dep = name if deps[name] in ('', '*') else '{} {}'.format(
name, deps[name])
doc['dependencies'][index] = dep
# add new deps
for name, version in sorted(deps.items()):
if name in represented:
continue
dep = name if version in ('',
'*') else '{} {}'.format(name, version)
doc['dependencies'].append(dep)
# drop empty section
if not doc['dependencies']:
del doc['dependencies']
stream = StringIO()
yaml_dump(doc, stream)
stream.seek(0)
return stream.read()
def summary(self):
s = StringIO()
print(json.dumps(self.to_log_dict(), indent="\t"), file=s)
s.seek(0)
return s.read()
def generateR(url):
notPlot = False
netcdf = False
data_type = "plot"
try:
url = json.loads(url)
fileExtension = ".png"
except:
notPlot = True
url_tail = "&" + re.findall("&(.*)", url)[0]
url = re.sub("&(.*)", "", url)
url = json.loads(url)
fileExtension = ".csv"
if 'output_format' in url:
netcdf = True
fileExtension = ".nc"
data_type = "subset"
#setup file
script = StringIO()
#FILE CONTENTS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
script.write("#Assigns Base Url\n\n")
script.write('base_url <- "http://navigator.oceansdata.ca/api/v1.0/' +
data_type + '/"\n\n\n')
old_query = dict(url)
#Assign Query Variables
script.write("#Assignes Query Variables\n")
for x in url:
updated_value = re.sub("'", '"', str(url.get(x)))
updated_value = re.sub("True", "true", updated_value)
updated_value = re.sub("False", "false", updated_value)
if isinstance(url.get(x), str):
script.write(x + '= ' "'" + '"' + updated_value + '"' + "'\n")
elif isinstance(url.get(x), bool):
script.write(x + '= "' + updated_value + '"\n')
elif isinstance(url.get(x), int):
if x == 'time':
formatted_date = time_query_conversion(url.get('dataset'),
url.get('time'))
script.write(x + '= ' + "'" + '"' + formatted_date + '"' +
"'" + '\n')
else:
script.write(x + '= ' + updated_value + '\n')
else:
script.write(x + '= ' + "'" + updated_value + "'\n")
script.write("\n")
if notPlot == True:
script.write("url_tail <- '" + url_tail + "'\n")
#Assemble Query
queryVariables = ''
query_template = url
script.write("#Assembles Query Using the Assigned Values\n")
script.write("query = sprintf('")
for x in url:
url[x] = '%s'
#query_template = re.sub(str(url.get(x)), '%s', str(query_template), 1)
queryVariables += ', ' + x
query_template = str(query_template)
query_template = re.sub("'%s'", "%s", query_template)
query_template = re.sub(
"'", '"', query_template) #Replace signle quotes with double quotes
script.write(query_template)
#queryVariables = re.sub(',', '', queryVariables, 1)
script.write("'")
script.write(queryVariables)
script.write(')\n\n')
#Request and Save Image
script.write("#Request and Save Image\n")
if notPlot == False:
script.write('full_url <- paste0(base_url, "?query=", query)\n')
else:
script.write(
'full_url <- paste0(base_url, "?query=", query, url_tail)\n')
if old_query.get("type") == "drifter":
script.write(
'filename = paste0("script_template_", dataset, "_", drifter, ".png")\n'
)
else:
script.write("#Format time to be used in file name\n")
script.write("time_ = gsub(':.*','', time)\n")
script.write("time_ = gsub('\\" + '"' + "'," + '"",' + "time_)\n\n")
script.write('filename = paste0("script_output_", time_, "' +
fileExtension + '")\n')
script.write(
'download.file(full_url, filename, extra = "curl", mode = "wb")\n')
# EOF FILE CONTENT ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
b = BytesIO()
script.seek(0)
b.write(bytes(script.read(), 'utf-8'))
b.seek(0)
#Hash Result (For Testing)
#data = b.read()
#m = hashlib.md5()
#m.update(data)
#print(m.hexdigest())
#b.seek(0)
return b
def init_sqlite(self):
"""get sqlite connection giving path of database file.
args:
sqlite_file (str): path of sqlite database file.
return:
connection object
"""
sqlite_file = os.path.join(FLAGS.data_dir, DMOZ_SQLITE)
logging.info("get dmoz sqlite connection")
if os.path.isfile(sqlite_file):
logging.info("DB already exists: " + sqlite_file)
# conn = sqlite3.connect(sqlite_file, check_same_thread=False)
# Read database to tempfile
con = sqlite3.connect(sqlite_file)
tempfile = StringIO()
for line in con.iterdump():
tempfile.write('%s\n' % line)
con.close()
tempfile.seek(0)
# Create a database in memory and import from tempfile
conn = sqlite3.connect(":memory:", check_same_thread=False)
conn.cursor().executescript(tempfile.read())
conn.commit()
conn.row_factory = sqlite3.Row
else:
dmoz_json_path = os.path.join(FLAGS.data_dir, DMOZ_JSON)
if not os.path.isfile(dmoz_json_path):
raise FileNotFoundError(dmoz_json_path)
dmoz = read_json(dmoz_json_path)
logging.info("creating new database file: " + sqlite_file)
# enable Serialized, only read is used.
conn = sqlite3.connect(sqlite_file, check_same_thread=False)
# Create table
conn.execute('CREATE TABLE dmoz (url TEXT, category INTEGER)')
# conn.execute('CREATE TABLE dmoz (url TEXT PRIMARY KEY, category INTEGER)')
logging.info("created table")
row_ind = 0
url_chunk = []
for cat in dmoz:
for page in dmoz[cat]:
# url = urlparse(page['url'])
# # garentee / after netloc
# path = '/'
# if url.path:
# path = url.path
url_clean = _clean_url(page['url'])
url_chunk.append((url_clean, CATEGORIES.index(cat)))
row_ind += 1
if row_ind % FLAGS.insert_chunk_size == 0:
# sql insert many rows
# logging.debug("url_chunk: {}".format(url_chunk))
conn.executemany('INSERT INTO dmoz VALUES (?,?)',
url_chunk)
url_chunk = []
conn.commit()
logging.info(
"row {}: inset {} rows to dmoz TABLE".format(
row_ind, FLAGS.insert_chunk_size))
# insert the last block
if url_chunk:
conn.executemany('INSERT INTO dmoz VALUES (?,?)', url_chunk)
url_chunk = []
conn.commit()
logging.info("row {}: inset {} rows to dmoz TABLE".format(
row_ind, len(url_chunk)))
# create index for url
logging.info("creating url index for dmonz DB")
conn.execute('CREATE INDEX url_index ON dmoz (url)')
# def regexp(expr, item):
# reg = re.compile(expr)
# return reg.search(item) is not None
#
# conn.create_function("REGEXP", 2, regexp)
self.conn = conn
def generatePython(url):
notPlot = False
netcdf = False
data_type = "plot"
try:
url = json.loads(url)
except:
notPlot = True
url_tail = "&" + re.findall("&(.*)", url)[0]
url = re.sub("&(.*)", "", url)
url = json.loads(url)
fileExtension = "csv"
if 'output_format' in url:
netcdf = True
fileExtension = "nc"
data_type = "subset"
#setup file
script = StringIO()
#FILE CONTENTS~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#HEADER---------------------
if notPlot == True or netcdf == True:
script.write("import requests\n")
script.write("import os\n")
script.write("import shutil\n")
script.write("from urllib.request import urlopen\n")
script.write("from urllib.parse import urlencode\n")
script.write("from contextlib import closing\n")
script.write("try:\n")
script.write(" from PIL import Image\n")
script.write("except:\n")
script.write(
' print("If you are on a Windows machine, please install PIL (imaging library) using'
+ " 'python -m pip install Pillow'" + ' in the Anaconda Prompt")\n')
script.write(" exit()\n")
script.write("import json\n")
script.write("\n\n")
#Set Navigators URL
script.write("#Set Navigator URL\n")
script.write(
'base_url = "http://navigator.oceansdata.ca/api/v1.0/%s/?"\n\n' %
(data_type))
#---------------------------
#CREATE JSON OBJECT---------
script.write("#Create JSON Object\n")
script.write("query = {\n")
for x in url:
#print(type(url.get(x)))
if isinstance(url.get(x), str):
script.write(' "' + x + '": "' + str(url.get(x)) + '"' + ",\n")
else:
if x == 'time':
formatted_date = time_query_conversion(url.get('dataset'),
url.get('time'))
script.write(' "' + x + '": ' + "'" + formatted_date + "',\n")
else:
script.write(' "' + x + '": ' + str(url.get(x)) + ",\n")
script.write("}\n")
#---------------------------
#Assemble full request
script.write('\n#Assemble full request - converts json object to url\n')
if notPlot == False:
script.write("url = base_url + urlencode(" + '{"query": ' +
"json.dumps(query)})" + "\n")
else:
script.write("url = base_url + urlencode(" + '{"query": ' +
"json.dumps(query)}) + '" + url_tail + "'\n")
script.write("print(url)\n")
#Open URL and read response
if notPlot == False and netcdf == False:
script.write("\n#Open URL and save response\n")
script.write("with closing(urlopen(url)) as f:\n")
script.write(" img = Image.open(f)\n")
if url.get("type") == "drifter":
script.write(
' img.save("script_template_" + str(query["dataset"]) + "_" + str(query["drifter"]) + ".png", "PNG")\n'
)
else:
script.write(
' img.save("script_template_" + str(query["dataset"]) + "_" + str(query["variable"]) + ".png" , "PNG")\n'
)
else:
script.write("\n#Open URL and save response\n")
script.write("data_file = requests.get(url, stream=True)\n")
script.write("dump = data_file.raw\n")
script.write("location = os.path.abspath('/home/script_output.%s')\n" %
(fileExtension))
script.write('with open("script_output.%s", "wb") as location:\n' %
(fileExtension))
script.write(" print('Saving File')\n")
script.write(" shutil.copyfileobj(dump, location)\n")
script.write(" print('Done')\n")
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#CONVERT TO BytesIO TO SEND
b = BytesIO()
script.seek(0)
b.write(bytes(script.read(), 'utf-8'))
b.seek(0)
return b
def _run_dump(value) -> str:
output = StringIO()
dump(value, to=output)
output.seek(0)
return output.read()
class EmbeddedSphinxShell(object):
"""An embedded IPython instance to run inside Sphinx"""
def __init__(self, exec_lines=None):
self.cout = StringIO()
if exec_lines is None:
exec_lines = []
# Create config object for IPython
config = Config()
config.HistoryManager.hist_file = ':memory:'
config.InteractiveShell.autocall = False
config.InteractiveShell.autoindent = False
config.InteractiveShell.colors = 'NoColor'
# create a profile so instance history isn't saved
tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
profname = 'auto_profile_sphinx_build'
pdir = os.path.join(tmp_profile_dir, profname)
profile = ProfileDir.create_profile_dir(pdir)
# Create and initialize global ipython, but don't start its mainloop.
# This will persist across different EmbededSphinxShell instances.
IP = InteractiveShell.instance(config=config, profile_dir=profile)
atexit.register(self.cleanup)
# io.stdout redirect must be done after instantiating InteractiveShell
io.stdout = self.cout
io.stderr = self.cout
# For debugging, so we can see normal output, use this:
#from IPython.utils.io import Tee
#io.stdout = Tee(self.cout, channel='stdout') # dbg
#io.stderr = Tee(self.cout, channel='stderr') # dbg
# Store a few parts of IPython we'll need.
self.IP = IP
self.user_ns = self.IP.user_ns
self.user_global_ns = self.IP.user_global_ns
self.input = ''
self.output = ''
self.tmp_profile_dir = tmp_profile_dir
self.is_verbatim = False
self.is_doctest = False
self.is_suppress = False
# Optionally, provide more detailed information to shell.
# this is assigned by the SetUp method of IPythonDirective
# to point at itself.
#
# So, you can access handy things at self.directive.state
self.directive = None
# on the first call to the savefig decorator, we'll import
# pyplot as plt so we can make a call to the plt.gcf().savefig
self._pyplot_imported = False
# Prepopulate the namespace.
for line in exec_lines:
self.process_input_line(line, store_history=False)
def cleanup(self):
shutil.rmtree(self.tmp_profile_dir, ignore_errors=True)
def clear_cout(self):
self.cout.seek(0)
self.cout.truncate(0)
def process_input_line(self, line, store_history=True):
"""process the input, capturing stdout"""
stdout = sys.stdout
splitter = self.IP.input_splitter
try:
sys.stdout = self.cout
splitter.push(line)
more = splitter.push_accepts_more()
if not more:
source_raw = splitter.raw_reset()
self.IP.run_cell(source_raw, store_history=store_history)
finally:
sys.stdout = stdout
def process_image(self, decorator):
"""
# build out an image directive like
# .. image:: somefile.png
# :width 4in
#
# from an input like
# savefig somefile.png width=4in
"""
savefig_dir = self.savefig_dir
source_dir = self.source_dir
saveargs = decorator.split(' ')
filename = saveargs[1]
# insert relative path to image file in source
outfile = os.path.relpath(os.path.join(savefig_dir, filename),
source_dir)
imagerows = ['.. image:: %s' % outfile]
for kwarg in saveargs[2:]:
arg, val = kwarg.split('=')
arg = arg.strip()
val = val.strip()
imagerows.append(' :%s: %s' % (arg, val))
image_file = os.path.basename(outfile) # only return file name
image_directive = '\n'.join(imagerows)
return image_file, image_directive
# Callbacks for each type of token
def process_input(self, data, input_prompt, lineno):
"""
Process data block for INPUT token.
"""
decorator, input, rest = data
image_file = None
image_directive = None
is_verbatim = decorator == '@verbatim' or self.is_verbatim
is_doctest = (decorator is not None and \
decorator.startswith('@doctest')) or self.is_doctest
is_suppress = decorator == '@suppress' or self.is_suppress
is_okexcept = decorator == '@okexcept' or self.is_okexcept
is_okwarning = decorator == '@okwarning' or self.is_okwarning
is_savefig = decorator is not None and \
decorator.startswith('@savefig')
input_lines = input.split('\n')
if len(input_lines) > 1:
if input_lines[-1] != "":
input_lines.append('') # make sure there's a blank line
# so splitter buffer gets reset
continuation = ' %s:' % ''.join(['.'] * (len(str(lineno)) + 2))
if is_savefig:
image_file, image_directive = self.process_image(decorator)
ret = []
is_semicolon = False
# Hold the execution count, if requested to do so.
if is_suppress and self.hold_count:
store_history = False
else:
store_history = True
# Note: catch_warnings is not thread safe
with warnings.catch_warnings(record=True) as ws:
for i, line in enumerate(input_lines):
if line.endswith(';'):
is_semicolon = True
if i == 0:
# process the first input line
if is_verbatim:
self.process_input_line('')
self.IP.execution_count += 1 # increment it anyway
else:
# only submit the line in non-verbatim mode
self.process_input_line(line,
store_history=store_history)
formatted_line = '%s %s' % (input_prompt, line)
else:
# process a continuation line
if not is_verbatim:
self.process_input_line(line,
store_history=store_history)
formatted_line = '%s %s' % (continuation, line)
if not is_suppress:
ret.append(formatted_line)
if not is_suppress and len(rest.strip()) and is_verbatim:
# The "rest" is the standard output of the input. This needs to be
# added when in verbatim mode. If there is no "rest", then we don't
# add it, as the new line will be added by the processed output.
ret.append(rest)
# Fetch the processed output. (This is not the submitted output.)
self.cout.seek(0)
processed_output = self.cout.read()
if not is_suppress and not is_semicolon:
#
# In IPythonDirective.run, the elements of `ret` are eventually
# combined such that '' entries correspond to newlines. So if
# `processed_output` is equal to '', then the adding it to `ret`
# ensures that there is a blank line between consecutive inputs
# that have no outputs, as in:
#
# In [1]: x = 4
#
# In [2]: x = 5
#
# When there is processed output, it has a '\n' at the tail end. So
# adding the output to `ret` will provide the necessary spacing
# between consecutive input/output blocks, as in:
#
# In [1]: x
# Out[1]: 5
#
# In [2]: x
# Out[2]: 5
#
# When there is stdout from the input, it also has a '\n' at the
# tail end, and so this ensures proper spacing as well. E.g.:
#
# In [1]: print x
# 5
#
# In [2]: x = 5
#
# When in verbatim mode, `processed_output` is empty (because
# nothing was passed to IP. Sometimes the submitted code block has
# an Out[] portion and sometimes it does not. When it does not, we
# need to ensure proper spacing, so we have to add '' to `ret`.
# However, if there is an Out[] in the submitted code, then we do
# not want to add a newline as `process_output` has stuff to add.
# The difficulty is that `process_input` doesn't know if
# `process_output` will be called---so it doesn't know if there is
# Out[] in the code block. The requires that we include a hack in
# `process_block`. See the comments there.
#
ret.append(processed_output)
elif is_semicolon:
# Make sure there is a newline after the semicolon.
ret.append('')
# context information
filename = "Unknown"
lineno = 0
if self.directive.state:
filename = self.directive.state.document.current_source
lineno = self.directive.state.document.current_line
# output any exceptions raised during execution to stdout
# unless :okexcept: has been specified.
if not is_okexcept and "Traceback" in processed_output:
s = "\nException in %s at block ending on line %s\n" % (filename,
lineno)
s += "Specify :okexcept: as an option in the ipython:: block to suppress this message\n"
sys.stdout.write('\n\n>>>' + ('-' * 73))
sys.stdout.write(s)
sys.stdout.write(processed_output)
sys.stdout.write('<<<' + ('-' * 73) + '\n\n')
# output any warning raised during execution to stdout
# unless :okwarning: has been specified.
if not is_okwarning:
for w in ws:
s = "\nWarning in %s at block ending on line %s\n" % (filename,
lineno)
s += "Specify :okwarning: as an option in the ipython:: block to suppress this message\n"
sys.stdout.write('\n\n>>>' + ('-' * 73))
sys.stdout.write(s)
sys.stdout.write(('-' * 76) + '\n')
s = warnings.formatwarning(w.message, w.category, w.filename,
w.lineno, w.line)
sys.stdout.write(s)
sys.stdout.write('<<<' + ('-' * 73) + '\n')
self.cout.truncate(0)
return (ret, input_lines, processed_output, is_doctest, decorator,
image_file, image_directive)
def process_output(self, data, output_prompt, input_lines, output,
is_doctest, decorator, image_file):
"""
Process data block for OUTPUT token.
"""
# Recall: `data` is the submitted output, and `output` is the processed
# output from `input_lines`.
TAB = ' ' * 4
if is_doctest and output is not None:
found = output # This is the processed output
found = found.strip()
submitted = data.strip()
if self.directive is None:
source = 'Unavailable'
content = 'Unavailable'
else:
source = self.directive.state.document.current_source
content = self.directive.content
# Add tabs and join into a single string.
content = '\n'.join([TAB + line for line in content])
# Make sure the output contains the output prompt.
ind = found.find(output_prompt)
if ind < 0:
e = ('output does not contain output prompt\n\n'
'Document source: {0}\n\n'
'Raw content: \n{1}\n\n'
'Input line(s):\n{TAB}{2}\n\n'
'Output line(s):\n{TAB}{3}\n\n')
e = e.format(source,
content,
'\n'.join(input_lines),
repr(found),
TAB=TAB)
raise RuntimeError(e)
found = found[len(output_prompt):].strip()
# Handle the actual doctest comparison.
if decorator.strip() == '@doctest':
# Standard doctest
if found != submitted:
e = ('doctest failure\n\n'
'Document source: {0}\n\n'
'Raw content: \n{1}\n\n'
'On input line(s):\n{TAB}{2}\n\n'
'we found output:\n{TAB}{3}\n\n'
'instead of the expected:\n{TAB}{4}\n\n')
e = e.format(source,
content,
'\n'.join(input_lines),
repr(found),
repr(submitted),
TAB=TAB)
raise RuntimeError(e)
else:
self.custom_doctest(decorator, input_lines, found, submitted)
# When in verbatim mode, this holds additional submitted output
# to be written in the final Sphinx output.
# https://github.com/ipython/ipython/issues/5776
out_data = []
is_verbatim = decorator == '@verbatim' or self.is_verbatim
if is_verbatim and data.strip():
# Note that `ret` in `process_block` has '' as its last element if
# the code block was in verbatim mode. So if there is no submitted
# output, then we will have proper spacing only if we do not add
# an additional '' to `out_data`. This is why we condition on
# `and data.strip()`.
# The submitted output has no output prompt. If we want the
# prompt and the code to appear, we need to join them now
# instead of adding them separately---as this would create an
# undesired newline. How we do this ultimately depends on the
# format of the output regex. I'll do what works for the default
# prompt for now, and we might have to adjust if it doesn't work
# in other cases. Finally, the submitted output does not have
# a trailing newline, so we must add it manually.
out_data.append("{0} {1}\n".format(output_prompt, data))
return out_data
def process_comment(self, data):
"""Process data fPblock for COMMENT token."""
if not self.is_suppress:
return [data]
def save_image(self, image_file):
"""
Saves the image file to disk.
"""
self.ensure_pyplot()
command = 'plt.gcf().savefig("%s")' % image_file
#print 'SAVEFIG', command # dbg
self.process_input_line('bookmark ipy_thisdir', store_history=False)
self.process_input_line('cd -b ipy_savedir', store_history=False)
self.process_input_line(command, store_history=False)
self.process_input_line('cd -b ipy_thisdir', store_history=False)
self.process_input_line('bookmark -d ipy_thisdir', store_history=False)
self.clear_cout()
def process_block(self, block):
"""
process block from the block_parser and return a list of processed lines
"""
ret = []
output = None
input_lines = None
lineno = self.IP.execution_count
input_prompt = self.promptin % lineno
output_prompt = self.promptout % lineno
image_file = None
image_directive = None
found_input = False
for token, data in block:
if token == COMMENT:
out_data = self.process_comment(data)
elif token == INPUT:
found_input = True
(out_data, input_lines, output, is_doctest,
decorator, image_file, image_directive) = \
self.process_input(data, input_prompt, lineno)
elif token == OUTPUT:
if not found_input:
TAB = ' ' * 4
linenumber = 0
source = 'Unavailable'
content = 'Unavailable'
if self.directive:
linenumber = self.directive.state.document.current_line
source = self.directive.state.document.current_source
content = self.directive.content
# Add tabs and join into a single string.
content = '\n'.join([TAB + line for line in content])
e = ('\n\nInvalid block: Block contains an output prompt '
'without an input prompt.\n\n'
'Document source: {0}\n\n'
'Content begins at line {1}: \n\n{2}\n\n'
'Problematic block within content: \n\n{TAB}{3}\n\n')
e = e.format(source, linenumber, content, block, TAB=TAB)
# Write, rather than include in exception, since Sphinx
# will truncate tracebacks.
sys.stdout.write(e)
raise RuntimeError('An invalid block was detected.')
out_data = \
self.process_output(data, output_prompt, input_lines,
output, is_doctest, decorator,
image_file)
if out_data:
# Then there was user submitted output in verbatim mode.
# We need to remove the last element of `ret` that was
# added in `process_input`, as it is '' and would introduce
# an undesirable newline.
assert (ret[-1] == '')
del ret[-1]
if out_data:
ret.extend(out_data)
# save the image files
if image_file is not None:
self.save_image(image_file)
return ret, image_directive
def ensure_pyplot(self):
"""
Ensures that pyplot has been imported into the embedded IPython shell.
Also, makes sure to set the backend appropriately if not set already.
"""
# We are here if the @figure pseudo decorator was used. Thus, it's
# possible that we could be here even if python_mplbackend were set to
# `None`. That's also strange and perhaps worthy of raising an
# exception, but for now, we just set the backend to 'agg'.
if not self._pyplot_imported:
if 'matplotlib.backends' not in sys.modules:
# Then ipython_matplotlib was set to None but there was a
# call to the @figure decorator (and ipython_execlines did
# not set a backend).
#raise Exception("No backend was set, but @figure was used!")
import matplotlib
matplotlib.use('agg')
# Always import pyplot into embedded shell.
self.process_input_line('import matplotlib.pyplot as plt',
store_history=False)
self._pyplot_imported = True
def process_pure_python(self, content):
"""
content is a list of strings. it is unedited directive content
This runs it line by line in the InteractiveShell, prepends
prompts as needed capturing stderr and stdout, then returns
the content as a list as if it were ipython code
"""
output = []
savefig = False # keep up with this to clear figure
multiline = False # to handle line continuation
multiline_start = None
fmtin = self.promptin
ct = 0
for lineno, line in enumerate(content):
line_stripped = line.strip()
if not len(line):
output.append(line)
continue
# handle decorators
if line_stripped.startswith('@'):
output.extend([line])
if 'savefig' in line:
savefig = True # and need to clear figure
continue
# handle comments
if line_stripped.startswith('#'):
output.extend([line])
continue
# deal with lines checking for multiline
continuation = u' %s:' % ''.join(['.'] * (len(str(ct)) + 2))
if not multiline:
modified = u"%s %s" % (fmtin % ct, line_stripped)
output.append(modified)
ct += 1
try:
ast.parse(line_stripped)
output.append(u'')
except Exception: # on a multiline
multiline = True
multiline_start = lineno
else: # still on a multiline
modified = u'%s %s' % (continuation, line)
output.append(modified)
# if the next line is indented, it should be part of multiline
if len(content) > lineno + 1:
nextline = content[lineno + 1]
if len(nextline) - len(nextline.lstrip()) > 3:
continue
try:
mod = ast.parse('\n'.join(content[multiline_start:lineno +
1]))
if isinstance(mod.body[0], ast.FunctionDef):
# check to see if we have the whole function
for element in mod.body[0].body:
if isinstance(element, ast.Return):
multiline = False
else:
output.append(u'')
multiline = False
except Exception:
pass
if savefig: # clear figure if plotted
self.ensure_pyplot()
self.process_input_line('plt.clf()', store_history=False)
self.clear_cout()
savefig = False
return output
def custom_doctest(self, decorator, input_lines, found, submitted):
"""
Perform a specialized doctest.
"""
from .custom_doctests import doctests
args = decorator.split()
doctest_type = args[1]
if doctest_type in doctests:
doctests[doctest_type](self, args, input_lines, found, submitted)
else:
e = "Invalid option to @doctest: {0}".format(doctest_type)
raise Exception(e)
class CodeEval(Cog):
"""Owner and admin feature that evaluates code and returns the result to the channel."""
def __init__(self, bot: Bot):
self.bot = bot
self.env = {}
self.ln = 0
self.stdout = StringIO()
self.interpreter = Interpreter(bot)
def _format(self, inp: str,
out: Any) -> Tuple[str, Optional[discord.Embed]]:
"""Format the eval output into a string & attempt to format it into an Embed."""
self._ = out
res = ""
# Erase temp input we made
if inp.startswith("_ = "):
inp = inp[4:]
# Get all non-empty lines
lines = [line for line in inp.split("\n") if line.strip()]
if len(lines) != 1:
lines += [""]
# Create the input dialog
for i, line in enumerate(lines):
if i == 0:
# Start dialog
start = f"In [{self.ln}]: "
else:
# Indent the 3 dots correctly;
# Normally, it's something like
# In [X]:
# ...:
#
# But if it's
# In [XX]:
# ...:
#
# You can see it doesn't look right.
# This code simply indents the dots
# far enough to align them.
# we first `str()` the line number
# then we get the length
# and use `str.rjust()`
# to indent it.
start = "...: ".rjust(len(str(self.ln)) + 7)
if i == len(lines) - 2:
if line.startswith("return"):
line = line[6:].strip()
# Combine everything
res += (start + line + "\n")
self.stdout.seek(0)
text = self.stdout.read()
self.stdout.close()
self.stdout = StringIO()
if text:
res += (text + "\n")
if out is None:
# No output, return the input statement
return (res, None)
res += f"Out[{self.ln}]: "
if isinstance(out, discord.Embed):
# We made an embed? Send that as embed
res += "<Embed>"
res = (res, out)
else:
if (isinstance(out, str) and
out.startswith("Traceback (most recent call last):\n")):
# Leave out the traceback message
out = "\n" + "\n".join(out.split("\n")[1:])
if isinstance(out, str):
pretty = out
else:
pretty = pprint.pformat(out, compact=True, width=60)
if pretty != str(out):
# We're using the pretty version, start on the next line
res += "\n"
if pretty.count("\n") > 20:
# Text too long, shorten
li = pretty.split("\n")
pretty = (
"\n".join(li[:3]) # First 3 lines
+ "\n ...\n" # Ellipsis to indicate removed lines
+ "\n".join(li[-3:])) # last 3 lines
# Add the output
res += pretty
res = (res, None)
return res # Return (text, embed)
async def _eval(self, ctx: Context,
code: str) -> Optional[discord.Message]:
"""Eval the input code string & send an embed to the invoking context."""
self.ln += 1
if code.startswith("exit"):
self.ln = 0
self.env = {}
return await ctx.send("```Reset history!```")
env = {
"message": ctx.message,
"author": ctx.message.author,
"channel": ctx.channel,
"guild": ctx.guild,
"ctx": ctx,
"self": self,
"bot": self.bot,
"inspect": inspect,
"discord": discord,
"contextlib": contextlib
}
self.env.update(env)
# Ignore this code, it works
_code = """
async def func(): # (None,) -> Any
try:
with contextlib.redirect_stdout(self.stdout):
{0}
if '_' in locals():
if inspect.isawaitable(_):
_ = await _
return _
finally:
self.env.update(locals())
""".format(textwrap.indent(code, ' '))
try:
exec(_code, self.env) # noqa: B102,S102
func = self.env['func']
res = await func()
except Exception:
res = traceback.format_exc()
out, embed = self._format(code, res)
await ctx.send(f"```py\n{out}```", embed=embed)
@group(name='internal', aliases=('int', ))
@with_role(Roles.owner, Roles.admin)
async def internal_group(self, ctx: Context) -> None:
"""Internal commands. Top secret!"""
if not ctx.invoked_subcommand:
await ctx.invoke(self.bot.get_command("help"), "internal")
@internal_group.command(name='eval', aliases=('e', ))
@with_role(Roles.admin, Roles.owner)
async def eval(self, ctx: Context, *, code: str) -> None:
"""Run eval in a REPL-like format."""
code = code.strip("`")
if re.match('py(thon)?\n', code):
code = "\n".join(code.split("\n")[1:])
if not re.search( # Check if it's an expression
r"^(return|import|for|while|def|class|"
r"from|exit|[a-zA-Z0-9]+\s*=)", code, re.M) and len(
code.split("\n")) == 1:
code = "_ = " + code
await self._eval(ctx, code)
def dumps(investigation):
# build MSI section
metadata_DF = pd.DataFrame(
columns=("Submission Title", "Submission Identifier",
"Submission Description", "Submission Version",
"Submission Reference Layer", "Submission Release Date",
"Submission Update Date"))
iversion_hits = [
x for x in investigation.comments if x.name == "Submission Version"
]
if len(iversion_hits) == 1:
investigation_version = iversion_hits[0].value
else:
investigation_version = ""
ireference_layer_hits = [
x for x in investigation.comments
if x.name == "Submission Reference Layer"
]
if len(ireference_layer_hits) == 1:
investigation_reference_layer = ireference_layer_hits[0].value
else:
investigation_reference_layer = ""
iversion_update_date = [
x for x in investigation.comments if x.name == "Submission Update Date"
]
if len(iversion_update_date) == 1:
investigation_update_date = iversion_update_date[0].value
else:
investigation_update_date = ""
metadata_DF.loc[0] = [
investigation.title, investigation.identifier,
investigation.description, investigation_version,
investigation_reference_layer, investigation.submission_date,
investigation_update_date
]
org_DF = pd.DataFrame(columns=("Organization Name", "Organization Address",
"Organization URI", "Organization Email",
"Organization Role"))
org_name_hits = [
x for x in investigation.comments
if x.name.startswith("Organization Name")
]
org_address_hits = [
x for x in investigation.comments
if x.name.startswith("Organization Address")
]
org_uri_hits = [
x for x in investigation.comments
if x.name.startswith("Organization URI")
]
org_email_hits = [
x for x in investigation.comments
if x.name.startswith("Organization Email")
]
org_role_hits = [
x for x in investigation.comments
if x.name.startswith("Organization Role")
]
for i, org_name in enumerate(org_name_hits):
try:
org_name = org_name_hits[i].value
except IndexError:
org_name = ""
try:
org_address = org_address_hits[i].value
except IndexError:
org_address = ""
try:
org_uri = org_uri_hits[i].value
except IndexError:
org_uri = ""
try:
org_email = org_email_hits[i].value
except IndexError:
org_email = ""
try:
org_role = org_role_hits[i].value
except IndexError:
org_role = ""
org_DF.loc[i] = [org_name, org_address, org_uri, org_email, org_role]
people_DF = pd.DataFrame(columns=("Person Last Name", "Person Initials",
"Person First Name", "Person Email",
"Person Role"))
for i, contact in enumerate(investigation.contacts):
if len(contact.roles) == 1:
role = contact.roles[0].term
else:
role = ""
people_DF.loc[i] = [
contact.last_name, contact.mid_initials, contact.first_name,
contact.email, role
]
term_sources_DF = pd.DataFrame(columns=("Term Source Name",
"Term Source URI",
"Term Source Version"))
for i, term_source in enumerate(investigation.ontology_source_references):
term_sources_DF.loc[i] = [
term_source.name, term_source.file, term_source.version
]
msi_DF = pd.concat([metadata_DF, org_DF, people_DF, term_sources_DF],
axis=1)
msi_DF = msi_DF.set_index("Submission Title").T
msi_DF = msi_DF.replace('', np.nan)
msi_memf = StringIO()
msi_DF.to_csv(path_or_buf=msi_memf,
index=True,
sep='\t',
encoding='utf-8',
index_label="Submission Title")
msi_memf.seek(0)
scd_DF = pd.DataFrame(columns=("Sample Name", "Sample Accession",
"Sample Description", "Derived From",
"Group Name", "Group Accession"))
all_samples = []
for study in investigation.studies:
all_samples += study.materials['sources']
all_samples += study.materials['samples']
all_samples = list(set(all_samples))
pbar = ProgressBar(min_value=0,
max_value=len(all_samples),
widgets=[
'Writing {} samples: '.format(len(all_samples)),
SimpleProgress(),
Bar(left=" |", right="| "),
ETA()
]).start()
for i, s in pbar(enumerate(all_samples)):
derived_from = ""
if isinstance(s, Sample) and s.derives_from is not None:
if len(s.derives_from) == 1:
derived_from_obj = s.derives_from[0]
derives_from_accession_hits = [
x for x in derived_from_obj.characteristics
if x.category.term == "Sample Accession"
]
if len(derives_from_accession_hits) == 1:
derived_from = derives_from_accession_hits[0].value
else:
print(
"WARNING! No Sample Accession available so referencing Derived From relation using Sample Name \"{}\" instead"
.format(derived_from_obj.name))
derived_from = derived_from_obj.name
sample_accession_hits = [
x for x in s.characteristics
if x.category.term == "Sample Accession"
]
if len(sample_accession_hits) == 1:
sample_accession = sample_accession_hits[0].value
else:
sample_accession = ""
sample_description_hits = [
x for x in s.characteristics
if x.category.term == "Sample Description"
]
if len(sample_description_hits) == 1:
sample_description = sample_description_hits[0].value
else:
sample_description = ""
if isinstance(s, Sample):
group_name_hits = [
x for x in s.factor_values
if x.factor_name.name == "Group Name"
]
if len(group_name_hits) == 1:
group_name = group_name_hits[0].value
else:
group_name = ""
group_accession_hits = [
x for x in s.factor_values
if x.factor_name.name == "Group Accession"
]
if len(group_accession_hits) == 1:
group_accession = group_accession_hits[0].value
else:
group_accession = ""
else:
group_name_hits = [
x for x in s.characteristics if x.category.term == "Group Name"
]
if len(group_name_hits) == 1:
group_name = group_name_hits[0].value
else:
group_name = ""
group_accession_hits = [
x for x in s.characteristics
if x.category.term == "Group Accession"
]
if len(group_accession_hits) == 1:
group_accession = group_accession_hits[0].value
else:
group_accession = ""
scd_DF.loc[i, "Sample Name"] = s.name
scd_DF.loc[i, "Sample Accession"] = sample_accession
scd_DF.loc[i, "Sample Description"] = sample_description
scd_DF.loc[i, "Derived From"] = derived_from
scd_DF.loc[i, "Group Name"] = group_name
scd_DF.loc[i, "Group Accession"] = group_accession
characteristics = [
x for x in s.characteristics if x.category.term not in
["Sample Description", "Derived From", "Sample Accession"]
]
for characteristic in characteristics:
characteristic_label = "Characteristic[{}]".format(
characteristic.category.term)
if characteristic_label not in scd_DF.columns:
scd_DF[characteristic_label] = ""
for val_col in get_value_columns(characteristic_label,
characteristic):
scd_DF[val_col] = ""
if isinstance(characteristic.value,
(int, float)) and characteristic.unit:
if isinstance(characteristic.unit, OntologyAnnotation):
scd_DF.loc[i, characteristic_label] = characteristic.value
scd_DF.loc[i, characteristic_label +
".Unit"] = characteristic.unit.term
scd_DF.loc[i, characteristic_label + ".Unit.Term Source REF"]\
= characteristic.unit.term_source.name if characteristic.unit.term_source else ""
scd_DF.loc[i, characteristic_label + ".Unit.Term Accession Number"] = \
characteristic.unit.term_accession
else:
scd_DF.loc[i, characteristic_label] = characteristic.value
scd_DF.loc[i, characteristic_label +
".Unit"] = characteristic.unit
elif isinstance(characteristic.value, OntologyAnnotation):
scd_DF.loc[i, characteristic_label] = characteristic.value.term
scd_DF.loc[i, characteristic_label + ".Term Source REF"] = \
characteristic.value.term_source.name if characteristic.value.term_source else ""
scd_DF.loc[i, characteristic_label + ".Term Accession Number"] = \
characteristic.value.term_accession
else:
scd_DF.loc[i, characteristic_label] = characteristic.value
scd_DF = scd_DF.replace('', np.nan)
columns = list(scd_DF.columns)
for i, col in enumerate(columns):
if col.endswith("Term Source REF"):
columns[i] = "Term Source REF"
elif col.endswith("Term Accession Number"):
columns[i] = "Term Source ID"
elif col.endswith("Unit"):
columns[i] = "Unit"
scd_DF.columns = columns
scd_memf = StringIO()
scd_DF.to_csv(path_or_buf=scd_memf,
index=False,
sep='\t',
encoding='utf-8')
scd_memf.seek(0)
sampletab_memf = StringIO()
sampletab_memf.write("[MSI]\n")
for line in msi_memf:
sampletab_memf.write(line.rstrip() + '\n')
sampletab_memf.write("[SCD]\n")
for line in scd_memf:
sampletab_memf.write(line.rstrip() + '\n')
sampletab_memf.seek(0)
return sampletab_memf.read()
def drawInlineImage(self, image, x, y, width=None, height=None):
"""Draw a PIL Image into the specified rectangle. If width and
height are omitted, they are calculated from the image size.
Also allow file names as well as images. This allows a
caching mechanism"""
# print "drawInlineImage: x=%s, y=%s, width = %s, height=%s " % (x,y, width, height)
try:
from PIL import Image
except ImportError:
print('Python Imaging Library not available')
return
try:
import zlib
except ImportError:
print('zlib not available')
return
self._currentPageHasImages = 1
if type(image) == StringType:
if os.path.splitext(image)[1] in ['.jpg', '.JPG']:
#directly process JPEG files
#open file, needs some error handling!!
imageFile = open(image, 'rb')
info = self.readJPEGInfo(imageFile)
imgwidth, imgheight = info[0], info[1]
if info[2] == 1:
colorSpace = 'DeviceGray'
elif info[2] == 3:
colorSpace = 'DeviceRGB'
else: #maybe should generate an error, is this right for CMYK?
colorSpace = 'DeviceCMYK'
imageFile.seek(0) #reset file pointer
imagedata = []
imagedata.append('BI') # begin image
# this describes what is in the image itself
imagedata.append('/Width %0.4f /Height %0.4f' %
(info[0], info[1]))
imagedata.append('/BitsPerComponent 8')
imagedata.append('/ColorSpace /%s' % colorSpace)
imagedata.append('/Filter [ /ASCII85Decode /DCTDecode]')
imagedata.append('ID')
#write in blocks of (??) 60 characters per line to a list
compressed = imageFile.read()
encoded = pdfutils._AsciiBase85Encode(compressed)
outstream = StringIO(encoded)
dataline = outstream.read(60)
while dataline != "":
imagedata.append(dataline)
dataline = outstream.read(60)
imagedata.append('EI')
else:
if not pdfutils.cachedImageExists(image):
pdfutils.cacheImageFile(image)
#now we have one cached, slurp it in
cachedname = os.path.splitext(image)[0] + '.a85'
imagedata = open(cachedname, 'rb').readlines()
#trim off newlines...
imagedata = map(string.strip, imagedata)
#parse line two for width, height
words = string.split(imagedata[1])
imgwidth = string.atoi(words[1])
imgheight = string.atoi(words[3])
else:
#PIL Image
#work out all dimensions
myimage = image.convert('RGB')
imgwidth, imgheight = myimage.size
imagedata = []
imagedata.append('BI') # begin image
# this describes what is in the image itself
imagedata.append(
'/W %0.4f /H %0.4f /BPC 8 /CS /RGB /F [/A85 /Fl]' %
(imgwidth, imgheight))
imagedata.append('ID')
#use a flate filter and Ascii Base 85 to compress
raw = getattr(myimage, 'tobytes', myimage.tostring)()
assert len(
raw) == imgwidth * imgheight, "Wrong amount of data for image"
compressed = zlib.compress(raw) #this bit is very fast...
encoded = pdfutils._AsciiBase85Encode(
compressed) #...sadly this isn't
#write in blocks of (??) 60 characters per line to a list
outstream = StringIO(encoded)
dataline = outstream.read(60)
while dataline != "":
imagedata.append(dataline)
dataline = outstream.read(60)
imagedata.append('EI')
#now build the PDF for the image.
if not width:
width = imgwidth
if not height:
height = imgheight
# this says where and how big to draw it
#self._code.append('ET')
#self._code.append('q %0.4f 0 0 %0.4f %0.4f %0.4f cm' % (width, height, x, y+height))
if self.bottomup:
self._code.append('q %0.4f 0 0 %0.4f %0.4f %0.4f cm' %
(width, height, x, y))
else:
# multiply height by (-1) to overcome flip in coordinate system -cwl
self._code.append('q %0.4f 0 0 %0.4f %0.4f %0.4f cm' %
(width, -height, x, y + height))
self._code.extend(imagedata)
self._code.append('Q')
class Stdout:
"""PhpSploit framework's dedicated standard output wrapper,
supplying some enhancements, such as pattern coloration and
back logging.
The 'backlog' argument is defaultly set to False, it can be
enabled at initialisation if set to True, or enabled later
setting the instance's backlog attribute to an empty string.
NOTE: See module's help for more informations.
"""
def __init__(self, outfile=sys.__stdout__, backlog=False):
# get original stdout
self._orig_outfile = outfile
# just in case we wrap at runtime o the future,
# as we did with `colorama_wrapper` in the past
self.outfile = self._orig_outfile
# handle back logging
self._backlog = StringIO()
if backlog:
self._has_backlog = True
else:
self._has_backlog = False
# are colors supported ?
self._has_colors = ui.output.colors()
self._write_lock = False
def __del__(self):
"""Restore the original sys.stdout on Wrapper deletion"""
self._backlog.close()
# dirty hack when used before argparse on main file...
try:
sys.stdout = self._orig_outfile
except:
pass
def __getattr__(self, obj):
"""Fallback to original stdout objects for undefined methods"""
return getattr(self._orig_outfile, obj)
def _writeLn(self, line):
"""Process individual line morphing, and write it"""
# Process per platform newline transformation
if line.endswith('\r\n'):
line = line[:-2] + os.linesep
elif line.endswith('\n'):
line = line[:-1] + os.linesep
# special case: debug tag is only printed if VERBOSITY is True
# NOTE: considering that the python print() function does another
# write() to add line separator, we need a self._write_lock
# canary to block it if the previous message display aborted.
from core import session
if line.startswith("[#] ") and not session.Conf.VERBOSITY():
self._write_lock = True
return
if self._write_lock:
self._write_lock = False
if line == os.linesep:
return
line = process_tags(line) # handle tagged lines coloration
# Write line to stdout, and it's decolorized version on backlog
# if standard output is not a tty, decolorize anything.
if self._has_backlog:
self._backlog.write(decolorize(line))
if not self._has_colors:
line = decolorize(line)
try:
self.outfile.write(line)
except UnicodeEncodeError:
buf = encoding.encode(line)
self.outfile.buffer.write(buf)
def write(self, string):
"""Write the given string to stdout"""
for line in string.splitlines(1):
self._writeLn(line)
@property
def backlog(self):
"""A dedicated stdout back logging buffer"""
if self._has_backlog:
self._backlog.seek(0)
return self._backlog.read()
else:
raise AttributeError()
@backlog.setter
def backlog(self, value):
"""Setting backlog's value to None or False disables it,
While giving any other value resets the backlog buffer.
If a non empty string is given, backlog takes it as new value
"""
del self.backlog
if not (value is False or value is None):
self._has_backlog = True
if type(value) == str:
self._backlog.write(decolorize(value))
@backlog.deleter
def backlog(self):
"""Flush backlog buffer and mark it as disabled on deletion"""
self._backlog.truncate(0)
self._backlog.seek(0)
self._has_backlog = False
def _write_summary_log(self, upgrade_id: str,
requirements: UpgradeRequirements):
try:
Path(self.LOGS_DIR).mkdir(parents=True, exist_ok=True)
summary_text = StringIO()
summary_text.write('Upgrade summary ( id: {} )'.format(upgrade_id))
if requirements.cannot_upgrade:
summary_text.write('\nCannot upgrade:')
for dep in requirements.cannot_upgrade:
type_label = self.i18n.get(
'gem.{}.type.{}.label'.format(
dep.pkg.gem_name,
dep.pkg.get_type().lower()),
dep.pkg.get_type().capitalize())
summary_text.write(
'\n * Type:{}\tName: {}\tVersion: {}\tReason: {}'.
format(type_label, dep.pkg.name,
dep.pkg.version if dep.pkg.version else '?',
dep.reason if dep.reason else '?'))
summary_text.write('\n')
if requirements.to_remove:
summary_text.write('\nMust be removed:')
for dep in requirements.to_remove:
type_label = self.i18n.get(
'gem.{}.type.{}.label'.format(
dep.pkg.gem_name,
dep.pkg.get_type().lower()),
dep.pkg.get_type().capitalize())
summary_text.write(
'\n * Type:{}\tName: {}\tVersion: {}\tReason: {}'.
format(type_label, dep.pkg.name,
dep.pkg.version if dep.pkg.version else '?',
dep.reason if dep.reason else '?'))
summary_text.write('\n')
if requirements.to_install:
summary_text.write('\nMust be installed:')
for dep in requirements.to_install:
type_label = self.i18n.get(
'gem.{}.type.{}.label'.format(
dep.pkg.gem_name,
dep.pkg.get_type().lower()),
dep.pkg.get_type().capitalize())
summary_text.write(
'\n * Type:{}\tName: {}\tVersion: {}\tReason: {}'.
format(type_label, dep.pkg.name,
dep.pkg.version if dep.pkg.version else '?',
dep.reason if dep.reason else '?'))
summary_text.write('\n')
if requirements.to_upgrade:
summary_text.write('\nWill be upgraded:')
for dep in requirements.to_upgrade:
type_label = self.i18n.get(
'gem.{}.type.{}.label'.format(
dep.pkg.gem_name,
dep.pkg.get_type().lower()),
dep.pkg.get_type().capitalize())
summary_text.write(
'\n * Type:{}\tName: {}\tVersion:{} \tNew version: {}'.
format(
type_label, dep.pkg.name,
dep.pkg.version if dep.pkg.version else '?',
dep.pkg.latest_version
if dep.pkg.latest_version else '?'))
summary_text.write('\n')
summary_text.seek(0)
with open(self.SUMMARY_FILE.format(upgrade_id), 'w+') as f:
f.write(summary_text.read())
except:
traceback.print_exc()
def _build_config_string(self, config):
sfile = StringIO()
config.write(sfile)
sfile.seek(0)
return sfile.read().strip()
# 获得完整f
print(f.getvalue())
# >>> hello world!
# 读取StringIO
f = StringIO("Hello!\nHi!\nGoodbye!")
while True:
s = f.readline()
if s == "":
break
print(s.strip())
# >>>
# Hello!
# Hi!
# Goodbye!
# StringIO操作的只能是str, 如果要操作二进制数据, 就需要使用BytesIO
# BytesIO实现了在内存中读写bytes, 我们创建一个BytesIO, 然后写入一些bytes
from io import BytesIO
f = BytesIO()
f.write("中文".encode("utf-8"))
print(f.getvalue())
# >>> b"\xe4\xb8\xad\xe6\x96\x87"
# 请注意, 写入的不是str, 而是经过UTF-8编码的bytes
from io import BytesIO
f = BytesIO(b"\xe4\xb8\xad\xe6\x96\x87")
print(f.read().decode("utf-8"))
# >>> 中文
def stream(nead_version, hashed_lines, model_class, display_values, null_value,
start, end, dict_fields, **kwargs):
# If kwargs 'start' and 'end' passed in URL validate and assign to dict_timestamps
dict_timestamps = {}
if '' not in [start, end]:
dict_timestamps = get_timestamp_iso_range_day_dict(start, end)
# Create buffer_ and writer objects
buffer_ = StringIO()
writer = csv.writer(buffer_, lineterminator="\n")
# Check if values passed for 'nead_version' and 'hashed_lines'
# If True: Write version and hash_lines to buffer_
if len(nead_version) > 0 and len(hashed_lines) > 0:
buffer_.writelines(nead_version)
buffer_.writelines(hashed_lines)
# Else: Write 'display_values' to buffer_
else:
buffer_.writelines(','.join(display_values) + '\n')
# Generator expressions to write each row in the queryset by calculating each row as needed and not all at once
# Write values that are null in database as the value assigned to 'null_value'
# Check if 'dict_fields' passed, if so stream aggregate daily data
if len(dict_fields) > 0:
queryset = model_class.objects \
.values_list('day') \
.annotate(**dict_fields) \
.filter(**dict_timestamps) \
.order_by('timestamp_first') \
.iterator()
for row in queryset:
# Call write_row
write_row(writer, null_value, row)
# Yield data (row from database)
buffer_.seek(0)
data = buffer_.read()
buffer_.seek(0)
buffer_.truncate()
yield data
# Elif kwargs 'start' and 'end' passed then apply timestamps filter
elif len(dict_timestamps) > 0:
queryset = model_class.objects \
.values_list(*display_values) \
.filter(**dict_timestamps) \
.order_by('timestamp_iso') \
.iterator()
for row in queryset:
# Call write_row
write_row(writer, null_value, row)
# Yield data (row from database)
buffer_.seek(0)
data = buffer_.read()
buffer_.seek(0)
buffer_.truncate()
yield data
# Elif retrieve all data currently in database table if 'display_values' passed and 'start' and 'end' are not passed
elif len(display_values) > 0:
queryset = model_class.objects \
.values_list(*display_values) \
.order_by('timestamp_iso') \
.iterator()
for row in queryset:
# Call write_row
write_row(writer, null_value, row)
# Yield data (row from database)
buffer_.seek(0)
data = buffer_.read()
buffer_.seek(0)
buffer_.truncate()
yield data
else:
raise FieldError(
"ERROR (stream.py) 'display_values' not passed in API call")
def _parse_entries(src):
entries = []
state = EXPECT_ENTRY
for lineno, line in enumerate(src.decode('utf-8').splitlines()):
# empty lines are skipped (but also terminate the notes section)
sline = line.strip()
if not sline or sline.startswith('#'):
state = EXPECT_ENTRY
continue
# non-empty line with leading spaces is interpreted as a notes line
if line[0] in [' ', '\t']:
if state != EXPECT_ENTRY_OR_NOTES:
raise SyntaxError(lineno, line, state)
# add line of notes
notes = entries[-1].notes
if notes:
notes += "\n"
notes += sline
entries[-1] = entries[-1]._replace(notes=notes)
continue
# parse line using shlex
sio = StringIO(line)
lexer = shlex(sio, posix=True)
lexer.whitespace_split = True
# otherwise, parse as an entry
try:
key = lexer.get_token()
except ValueError as e:
raise SyntaxError(lineno, line, str(e))
if not key:
raise SyntaxError(lineno, line, state)
key = key.rstrip(':')
try:
user = lexer.get_token()
except ValueError as e:
raise SyntaxError(lineno, line, str(e))
if not user:
raise SyntaxError(lineno, line, state)
user = user
try:
password = lexer.get_token()
except ValueError as e:
raise SyntaxError(lineno, line, str(e))
if not password:
password = user
user = notes = ''
else:
password = password
notes = sio.read().strip()
entries.append(Entry(key, user, password, notes))
state = EXPECT_ENTRY_OR_NOTES
return entries
from pylint import epylint as lint
from io import StringIO
import numpy
(pylint_stdout, pylint_stderr) = lint.py_run('test.py', return_std=True)
print(StringIO.read(pylint_stdout))
"""
StringIO
ATENÇÂO: Para ler ou escrever dados de um arquivo do sistema operacional o software
precisa ter permissão:
- Permissão de Leitura -> Para ler o arquivo.
- Permissão de Escrita -> Para Escrever no arquivo
StringIO -> Utilizado para ler e criar arquivos em memória.
"""
# Primeiro fazemos o import
from io import StringIO
mensagem = 'Esta é apenas uma string normal'
# Podemos criar um arquivo em memoria ja com uma string inserida ou mesmo vazio
# para inserirmos texto depois
arquivo = StringIO(mensagem)
print(arquivo.read())
arquivo.write(' Outro Texto')
arquivo.seek(0)
print(arquivo.read())
def _drawImageLevel1(self, image, x1, y1, x2=None, y2=None, **kwargs):
# Postscript Level1 version available for fallback mode when Level2 doesn't work
"""drawImage(self,image,x1,y1,x2=None,y2=None) : If x2 and y2 are omitted, they are
calculated from image size. (x1,y1) is upper left of image, (x2,y2) is lower right of
image in piddle coordinates."""
try:
from PIL import Image
except ImportError:
print('Python Imaging Library not available')
return
# For now let's start with 24 bit RGB images (following piddlePDF again)
print("Trying to drawImage in piddlePS")
component_depth = 8
myimage = image.convert('RGB')
imgwidth, imgheight = myimage.size
if not x2:
x2 = imgwidth + x1
if not y2:
y2 = y1 + imgheight
drawwidth = x2 - x1
drawheight = y2 - y1
print('Image size (%d, %d); Draw size (%d, %d)' % (imgwidth, imgheight, drawwidth, drawheight))
# now I need to tell postscript how big image is
# "image operators assume that they receive sample data from
# their data source in x-axis major index order. The coordinate
# of the lower-left corner of the first sample is (0,0), of the
# second (1,0) and so on" -PS2 ref manual p. 215
#
# The ImageMatrix maps unit square of user space to boundary of the source image
#
# The CurrentTransformationMatrix (CTM) maps the unit square of
# user space to the rect...on the page that is to receive the
# image. A common ImageMatrix is [width 0 0 -height 0 height]
# (for a left to right, top to bottom image )
# first let's map the user coordinates start at offset x1,y1 on page
self.code.extend([
'gsave',
'%s %s translate' % (x1, -y1 - drawheight), # need to start are lower left of image
'%s %s scale' % (drawwidth, drawheight),
'/scanline %d 3 mul string def' % imgwidth # scanline by multiples of image width
])
# now push the dimensions and depth info onto the stack
# and push the ImageMatrix to map the source to the target rectangle (see above)
# finally specify source (PS2 pp. 225 ) and by exmample
self.code.extend([
'%s %s %s' % (imgwidth, imgheight, component_depth),
'[%s %s %s %s %s %s]' % (imgwidth, 0, 0, -imgheight, 0, imgheight),
'{ currentfile scanline readhexstring pop } false 3', 'colorimage '
])
# data source output--now we just need to deliver a hex encode
# series of lines of the right overall size can follow
# piddlePDF again
rawimage = getattr(myimage, 'tobytes', myimage.tostring)()
assert len(rawimage) == imgwidth * imgheight, 'Wrong amount of data for image'
#compressed = zlib.compress(rawimage) # no zlib at moment
hex_encoded = self._AsciiHexEncode(rawimage)
# write in blocks of 78 chars per line
outstream = StringIO(hex_encoded)
dataline = outstream.read(78)
while dataline != "":
self.code.append(dataline)
dataline = outstream.read(78)
self.code.append('% end of image data') # for clarity
self.code.append('grestore') # return coordinates to normal
