def test_pickle_wrapper(self):
fh = BytesIO() # use cPickle with binary content
# test unwrapped results load save pickle
self.results._results.save(fh)
fh.seek(0, 0)
res_unpickled = self.results._results.__class__.load(fh)
assert type(res_unpickled) is type(self.results._results) # noqa:E721
# TODO: Check equality instead? This check isnt exactly meaningful
# test wrapped results load save
fh.seek(0, 0)
self.results.save(fh)
fh.seek(0, 0)
res_unpickled = self.results.__class__.load(fh)
fh.close()
assert type(res_unpickled) is type(self.results) # noqa:E721
# TODO: Check equality instead? This check isnt exactly meaningful
before = sorted(iterkeys(self.results.__dict__))
after = sorted(iterkeys(res_unpickled.__dict__))
assert before == after
before = sorted(iterkeys(self.results._results.__dict__))
after = sorted(iterkeys(res_unpickled._results.__dict__))
assert before == after
before = sorted(iterkeys(self.results.model.__dict__))
after = sorted(iterkeys(res_unpickled.model.__dict__))
assert before == after
before = sorted(iterkeys(self.results._cache))
after = sorted(iterkeys(res_unpickled._cache))
assert before == after
def __iter__(self):
if self.is_zipped:
byte_stream = BytesIO(self.response.content)
with zipfile.ZipFile(byte_stream) as self.zipfile:
for name in self.zipfile.namelist():
with self.zipfile.open(name) as single_file:
if name[-3:] == 'csv':
reader = self.get_csv_reader(single_file)
else:
reader = single_file
reader_iterator = iter(reader)
if self.is_header_present:
next(reader_iterator)
for line in reader_iterator:
yield self._parse_line(line)
byte_stream.close()
else:
stream = codecs.iterdecode(
self.response.iter_lines(), self.response.encoding
or self.response.apparent_encoding)
reader = csv.reader(stream, delimiter=self.delimiter)
reader_iterator = iter(reader)
if self.is_header_present:
next(reader_iterator)
for line in reader_iterator:
yield self._parse_line(line)
stream.close()
def getDataFromService(self, srvMethod, params):
self.infoMsg(
"Sending header request to %s" % self.getDestinationService(),
str(params))
result = self._sendTransferHeader(srvMethod, params)
if not result["OK"]:
self.errMsg("Could not send header", result["Message"])
return result
self.infoMsg("Starting to receive data from service")
_, srvTransport = result["Value"]
srvFileHelper = FileHelper(srvTransport)
srvFileHelper.setDirection("receive")
sIO = BytesIO()
result = srvFileHelper.networkToDataSink(sIO,
self.__transferBytesLimit)
if not result["OK"]:
self.errMsg("Could not receive data from server",
result["Message"])
srvTransport.close()
sIO.close()
return result
dataReceived = sIO.getvalue()
sIO.close()
self.infoMsg("Received %s bytes from service" % len(dataReceived))
retVal = srvTransport.receiveData()
srvTransport.close()
if not retVal["OK"]:
return retVal
return S_OK({"data": dataReceived, "srvResponse": retVal})
def updateBundles(self):
dirsToBundle = self.__getDirsToBundle()
# Delete bundles that don't have to be updated
for bId in self.__bundles:
if bId not in dirsToBundle:
gLogger.info("Deleting old bundle %s" % bId)
del(self.__bundles[bId])
for bId in dirsToBundle:
bundlePaths = dirsToBundle[bId]
gLogger.info("Updating %s bundle %s" % (bId, bundlePaths))
buffer_ = BytesIO()
filesToBundle = sorted(File.getGlobbedFiles(bundlePaths))
if filesToBundle:
commonPath = os.path.commonprefix(filesToBundle)
commonEnd = len(commonPath)
gLogger.info("Bundle will have %s files with common path %s" % (len(filesToBundle), commonPath))
with tarfile.open('dummy', "w:gz", buffer_) as tarBuffer:
for filePath in filesToBundle:
tarBuffer.add(filePath, filePath[commonEnd:])
zippedData = buffer_.getvalue()
buffer_.close()
hash_ = File.getMD5ForFiles(filesToBundle)
gLogger.info("Bundled %s : %s bytes (%s)" % (bId, len(zippedData), hash_))
self.__bundles[bId] = (hash_, zippedData)
else:
self.__bundles[bId] = (None, None)
def decompress(data):
"""
Decompresses the given data via the snappy algorithm.
If ``python-snappy`` is not installed a ``RuntimeError`` is raised.
"""
if not snappy_available:
raise RuntimeError("Snappy compression unavailable.")
buff_offset = len(raw_header) # skip the header
length = len(data) - len(raw_header)
output = BytesIO()
while buff_offset <= length:
block_size = struct.unpack_from("!i", data, buff_offset)[0]
buff_offset += struct.calcsize("!i")
block = struct.unpack_from("!%ds" % block_size, data, buff_offset)[0]
buff_offset += block_size
output.write(snappy.uncompress(block))
result = output.getvalue()
output.close()
return result
def compress(data):
"""
Compresses given data via the snappy algorithm.
The result is preceded with a header containing the string 'SNAPPY' and the
default and min-compat versions (both ``1``).
The block size for the compression is hard-coded at 32kb.
If ``python-snappy`` is not installed a ``RuntimeError`` is raised.
"""
if not snappy_available:
raise RuntimeError("Snappy compression unavailable.")
buff = BytesIO()
buff.write(raw_header)
for block_num in range(0, len(data), BLOCK_SIZE):
block = data[block_num:block_num + BLOCK_SIZE]
compressed = snappy.compress(block)
buff.write(struct.pack("!i", len(compressed)))
buff.write(compressed)
result = buff.getvalue()
buff.close()
return result
def serialize(
self, destination=None, encoding="utf-8", format='xml', **args):
if self.type in ('CONSTRUCT', 'DESCRIBE'):
return self.graph.serialize(
destination, encoding=encoding, format=format, **args)
"""stolen wholesale from graph.serialize"""
from rdflib import plugin
serializer = plugin.get(format, ResultSerializer)(self)
if destination is None:
stream = BytesIO()
stream2 = EncodeOnlyUnicode(stream)
serializer.serialize(stream2, encoding=encoding, **args)
return stream.getvalue()
if hasattr(destination, "write"):
stream = destination
serializer.serialize(stream, encoding=encoding, **args)
else:
location = destination
scheme, netloc, path, params, query, fragment = urlparse(location)
if netloc != "":
print("WARNING: not saving as location" +
"is not a local file reference")
return
fd, name = tempfile.mkstemp()
stream = os.fdopen(fd, 'wb')
serializer.serialize(stream, encoding=encoding, **args)
stream.close()
if hasattr(shutil, "move"):
shutil.move(name, path)
else:
shutil.copy(name, path)
os.remove(name)
def decompress(data):
"""
Decompresses the given data via the snappy algorithm.
If ``python-snappy`` is not installed a ``RuntimeError`` is raised.
"""
if not snappy_available:
raise RuntimeError("Snappy compression unavailable.")
buff_offset = len(raw_header) # skip the header
length = len(data) - len(raw_header)
output = BytesIO()
while buff_offset <= length:
block_size = struct.unpack_from("!i", data, buff_offset)[0]
buff_offset += struct.calcsize("!i")
block = struct.unpack_from("!%ds" % block_size, data, buff_offset)[0]
buff_offset += block_size
output.write(snappy.uncompress(block))
result = output.getvalue()
output.close()
return result
def __iter__(self):
if self.is_zipped:
byte_stream = BytesIO(self.response.content)
with zipfile.ZipFile(byte_stream) as self.zipfile:
for name in self.zipfile.namelist():
with self.zipfile.open(name) as single_file:
if name[-3:] == 'csv':
reader = csv.reader(single_file, delimiter=self.delimiter)
else:
reader = single_file
reader_iterator = iter(reader)
if self.is_header_present:
next(reader_iterator)
for line in reader_iterator:
yield self._parse_line(line)
byte_stream.close()
else:
stream = codecs.iterdecode(self.response.iter_lines(),
self.response.encoding or self.response.apparent_encoding)
reader = csv.reader(stream, delimiter=self.delimiter)
reader_iterator = iter(reader)
if self.is_header_present:
next(reader_iterator)
for line in reader_iterator:
yield self._parse_line(line)
stream.close()
def test_decode_response_gzip():
body = b'gzip message'
buf = BytesIO()
f = gzip.GzipFile('a', fileobj=buf, mode='wb')
f.write(body)
f.close()
compressed_body = buf.getvalue()
buf.close()
gzip_response = {
'body': {
'string': compressed_body
},
'headers': {
'access-control-allow-credentials': ['true'],
'access-control-allow-origin': ['*'],
'connection': ['keep-alive'],
'content-encoding': ['gzip'],
'content-length': ['177'],
'content-type': ['application/json'],
'date': ['Wed, 02 Dec 2015 19:44:32 GMT'],
'server': ['nginx']
},
'status': {
'code': 200,
'message': 'OK'
}
}
decoded_response = decode_response(gzip_response)
assert decoded_response['body']['string'] == body
assert decoded_response['headers']['content-length'] == [str(len(body))]
def visit_immutation(self, node, children):
context = self._final_context()
child_type = children[0].expr_name
if child_type == 'preview':
if self.tool == 'httpie':
command = ['http'] + context.httpie_args(self.method,
quote=True)
else:
assert self.tool == 'curl'
command = ['curl'] + context.curl_args(self.method, quote=True)
click.echo(' '.join(command))
elif child_type == 'action':
output = BytesIO()
try:
env = Environment(stdout=output, is_windows=False)
httpie_main(context.httpie_args(self.method), env=env)
content = output.getvalue()
finally:
output.close()
# XXX: Work around a bug of click.echo_via_pager(). When you pass
# a bytestring to echo_via_pager(), it converts the bytestring with
# str(b'abc'), which makes it "b'abc'".
if six.PY2:
content = unicode(content, 'utf-8') # noqa
else:
content = str(content, 'utf-8')
click.echo_via_pager(content)
return node
def __init__(self,
host,
port,
backend_mod=None,
pool=None,
is_ssl=False,
extra_headers=[],
proxy_pieces=None,
**ssl_args):
# connect the socket, if we are using an SSL connection, we wrap
# the socket.
self._s = backend_mod.Socket(socket.AF_INET, socket.SOCK_STREAM)
self._s.connect((host, port))
if proxy_pieces:
self._s.sendall(proxy_pieces)
response = StringIO()
while response.getvalue()[-4:] != b'\r\n\r\n':
response.write(self._s.recv(1))
response.close()
if is_ssl:
self._s = ssl.wrap_socket(self._s, **ssl_args)
self.extra_headers = extra_headers
self.is_ssl = is_ssl
self.backend_mod = backend_mod
self.host = host
self.port = port
self._connected = True
self._life = time.time() - random.randint(0, 10)
self._pool = pool
self._released = False
def output_properties(path=None,
content=None,
basename=None,
pseduo_location=False):
checksum = hashlib.sha1()
properties = {
"class": "File",
}
if path is not None:
properties["path"] = path
f = open(path, "rb")
else:
f = BytesIO(content)
try:
contents = f.read(1024 * 1024)
filesize = 0
while contents:
checksum.update(contents)
filesize += len(contents)
contents = f.read(1024 * 1024)
finally:
f.close()
properties["checksum"] = "sha1$%s" % checksum.hexdigest()
properties["size"] = filesize
set_basename_and_derived_properties(properties, basename)
_handle_pseudo_location(properties, pseduo_location)
return properties
def test_decode_response_gzip():
body = b'gzip message'
buf = BytesIO()
f = gzip.GzipFile('a', fileobj=buf, mode='wb')
f.write(body)
f.close()
compressed_body = buf.getvalue()
buf.close()
gzip_response = {
'body': {'string': compressed_body},
'headers': {
'access-control-allow-credentials': ['true'],
'access-control-allow-origin': ['*'],
'connection': ['keep-alive'],
'content-encoding': ['gzip'],
'content-length': ['177'],
'content-type': ['application/json'],
'date': ['Wed, 02 Dec 2015 19:44:32 GMT'],
'server': ['nginx']
},
'status': {'code': 200, 'message': 'OK'}
}
decoded_response = decode_response(gzip_response)
assert decoded_response['body']['string'] == body
assert decoded_response['headers']['content-length'] == [str(len(body))]
def test_augment_observation(test_file, test_file_uri):
test_fitsparser = FitsParser(test_file)
test_obs = Observation('collection', 'observation_id',
Algorithm('algorithm'))
test_fitsparser.augment_observation(test_obs, test_file_uri)
assert test_obs is not None
assert test_obs.planes is not None
assert len(test_obs.planes) == 1
test_plane = test_obs.planes['HI-line']
assert test_plane.artifacts is not None
assert len(test_plane.artifacts) == 1
test_artifact = test_plane.artifacts[test_file_uri]
assert test_artifact is not None
test_part = test_artifact.parts['0']
# remove the chunk bit, as it's part of other tests -
# results in <caom2:chunks/> xml output
test_part.chunks.pop()
# set the ids to expected values
test_obs._id = uuid.UUID('00000000000000001234567812345678')
test_plane._id = uuid.UUID('00000000000000001234567812345678')
test_artifact._id = uuid.UUID('00000000000000001234567812345678')
test_part._id = uuid.UUID('00000000000000001234567812345678')
output = BytesIO()
ow = ObservationWriter(False, False, "caom2",
obs_reader_writer.CAOM20_NAMESPACE)
ow.write(test_obs, output)
result = output.getvalue().decode('UTF-8')
output.close()
assert result == EXPECTED_OBS_XML # , result
class SSHStorageFile(File):
def __init__(self, name, storage, mode):
self._name = name
self._storage = storage
self._mode = mode
self._is_dirty = False
self.file = BytesIO()
self._size = None
@property
def size(self):
logger.debug("I am the size")
if not hasattr(self, '_size'):
self._size = self._storage.size(self._name)
return self._size
def read(self, num_bytes=None):
logger.debug("I am the read")
self.file = self._storage._read(self._name)
return self.file.read(num_bytes)
def write(self, content):
logger.debug("I am the write")
if 'w' not in self._mode:
raise AttributeError("File was opened for read-only access.")
self.file = BytesIO(content)
self._is_dirty = True
def close(self):
logger.debug("I am the close")
if self._is_dirty:
self._storage._save(self._name, self.file.getvalue())
self.file.close()
def visit_immutation(self, node, children):
context = self._final_context()
child_type = children[0].expr_name
if child_type == 'preview':
if self.tool == 'httpie':
command = ['http'] + context.httpie_args(self.method,
quote=True)
else:
assert self.tool == 'curl'
command = ['curl'] + context.curl_args(self.method, quote=True)
click.echo(' '.join(command))
elif child_type == 'action':
output = BytesIO()
try:
env = Environment(stdout=output, is_windows=False)
httpie_main(context.httpie_args(self.method), env=env)
content = output.getvalue()
finally:
output.close()
# XXX: Work around a bug of click.echo_via_pager(). When you pass
# a bytestring to echo_via_pager(), it converts the bytestring with
# str(b'abc'), which makes it "b'abc'".
if six.PY2:
content = unicode(content, 'utf-8') # noqa
else:
content = str(content, 'utf-8')
click.echo_via_pager(content)
return node
def compress(data):
"""
Compresses given data via the snappy algorithm.
The result is preceded with a header containing the string 'SNAPPY' and the
default and min-compat versions (both ``1``).
The block size for the compression is hard-coded at 32kb.
If ``python-snappy`` is not installed a ``RuntimeError`` is raised.
"""
if not snappy_available:
raise RuntimeError("Snappy compression unavailable.")
buff = BytesIO()
buff.write(raw_header)
for block_num in range(0, len(data), BLOCK_SIZE):
block = data[block_num:block_num + BLOCK_SIZE]
compressed = snappy.compress(block)
buff.write(struct.pack("!i", len(compressed)))
buff.write(compressed)
result = buff.getvalue()
buff.close()
return result
def test_writing_1_record(self):
expected = r"""
<?xml version="1.0" encoding="UTF-8"?>
<collection xmlns="http://www.loc.gov/MARC21/slim">
<record>
<leader> 22 4500</leader>
<datafield ind1="0" ind2="0" tag="100">
<subfield code="a">me</subfield>
</datafield>
<datafield ind1="0" ind2="0" tag="245">
<subfield code="a">Foo /</subfield>
<subfield code="c">by me.</subfield>
</datafield>
</record>
</collection>
"""
expected = textwrap.dedent(expected[1:]).replace('\n', '')
if str != binary_type:
expected = expected.encode()
file_handle = BytesIO()
try:
writer = pymarc.XMLWriter(file_handle)
record = pymarc.Record()
record.add_field(
pymarc.Field('100', ['0', '0'], ['a', u('me')]))
record.add_field(
pymarc.Field(
'245',
['0', '0'],
['a', u('Foo /'), 'c', u('by me.')]))
writer.write(record)
writer.close(close_fh=False)
self.assertEquals(file_handle.getvalue(), expected)
finally:
file_handle.close()
def test_write(self):
r = StringIO('ab')
n = FrenchNormalizer(r)
m = Matrix()
w = BytesIO()
m.feed(n)
m.dump(w)
expected = {
(None,): {
False: 1,
},
(False,): {
'a': 1,
},
('a',): {
'b': 1,
},
('b',): {
True: 1
}
}
print(pickle.loads(w.getvalue()))
print(expected)
assert pickle.loads(w.getvalue()) == expected
w.close()
def __init__(self, host, port, backend_mod=None, pool=None,
is_ssl=False, extra_headers=[], proxy_pieces=None, **ssl_args):
# connect the socket, if we are using an SSL connection, we wrap
# the socket.
self._s = backend_mod.Socket(socket.AF_INET, socket.SOCK_STREAM)
self._s.connect((host, port))
if proxy_pieces:
self._s.sendall(proxy_pieces)
response = StringIO()
while response.getvalue()[-4:] != b'\r\n\r\n':
response.write(self._s.recv(1))
response.close()
if is_ssl:
self._s = ssl.wrap_socket(self._s, **ssl_args)
self.extra_headers = extra_headers
self.is_ssl = is_ssl
self.backend_mod = backend_mod
self.host = host
self.port = port
self._connected = True
self._life = time.time() - random.randint(0, 10)
self._pool = pool
self._released = False
def output_properties(path=None, content=None, basename=None, pseduo_location=False):
checksum = hashlib.sha1()
properties = {
"class": "File",
}
if path is not None:
properties["path"] = path
f = open(path, "rb")
else:
f = BytesIO(content)
try:
contents = f.read(1024 * 1024)
filesize = 0
while contents:
checksum.update(contents)
filesize += len(contents)
contents = f.read(1024 * 1024)
finally:
f.close()
properties["checksum"] = "sha1$%s" % checksum.hexdigest()
properties["size"] = filesize
set_basename_and_derived_properties(properties, basename)
_handle_pseudo_location(properties, pseduo_location)
return properties
def test_writing_1_record(self):
expected = r"""
<?xml version="1.0" encoding="UTF-8"?>
<collection xmlns="http://www.loc.gov/MARC21/slim">
<record>
<leader> 22 4500</leader>
<datafield ind1="0" ind2="0" tag="100">
<subfield code="a">me</subfield>
</datafield>
<datafield ind1="0" ind2="0" tag="245">
<subfield code="a">Foo /</subfield>
<subfield code="c">by me.</subfield>
</datafield>
</record>
</collection>
"""
expected = textwrap.dedent(expected[1:]).replace('\n', '')
if str != binary_type:
expected = expected.encode()
file_handle = BytesIO()
try:
writer = pymarc.XMLWriter(file_handle)
record = pymarc.Record()
record.add_field(pymarc.Field('100', ['0', '0'], ['a', u('me')]))
record.add_field(
pymarc.Field(
'245', ['0', '0'],
['a', u('Foo /'), 'c', u('by me.')]))
writer.write(record)
writer.close(close_fh=False)
self.assertEquals(file_handle.getvalue(), expected)
finally:
file_handle.close()
def transfer_toClient(self, fileId, token, fileHelper):
version = ""
if isinstance(fileId, six.string_types):
if fileId in ['CAs', 'CRLs']:
return self.__transferFile(fileId, fileHelper)
else:
bId = fileId
elif isinstance(fileId, (list, tuple)):
if len(fileId) == 0:
fileHelper.markAsTransferred()
return S_ERROR("No bundle specified!")
elif len(fileId) == 1:
bId = fileId[0]
else:
bId = fileId[0]
version = fileId[1]
if not self.bundleManager.bundleExists(bId):
fileHelper.markAsTransferred()
return S_ERROR("Unknown bundle %s" % bId)
bundleVersion = self.bundleManager.getBundleVersion(bId)
if bundleVersion is None:
fileHelper.markAsTransferred()
return S_ERROR("Empty bundle %s" % bId)
if version == bundleVersion:
fileHelper.markAsTransferred()
return S_OK(bundleVersion)
buffer_ = BytesIO(self.bundleManager.getBundleData(bId))
result = fileHelper.DataSourceToNetwork(buffer_)
buffer_.close()
if not result['OK']:
return result
return S_OK(bundleVersion)
def test_decode_response_gzip():
body = b"gzip message"
buf = BytesIO()
f = gzip.GzipFile("a", fileobj=buf, mode="wb")
f.write(body)
f.close()
compressed_body = buf.getvalue()
buf.close()
gzip_response = {
"body": {
"string": compressed_body
},
"headers": {
"access-control-allow-credentials": ["true"],
"access-control-allow-origin": ["*"],
"connection": ["keep-alive"],
"content-encoding": ["gzip"],
"content-length": ["177"],
"content-type": ["application/json"],
"date": ["Wed, 02 Dec 2015 19:44:32 GMT"],
"server": ["nginx"],
},
"status": {
"code": 200,
"message": "OK"
},
}
decoded_response = decode_response(gzip_response)
assert decoded_response["body"]["string"] == body
assert decoded_response["headers"]["content-length"] == [str(len(body))]
def request(self, method, uri, xml_root=None):
"""
:type method: str
:type uri: str
:type xml_root: ETree.Element
:rtype: ETree.Element
:raises ClarityException: if Clarity returns an exception as XML
"""
request_start_seconds = time.perf_counter() if self.log_requests else 0
if xml_root is None:
response = self.raw_request(method, uri)
else:
# Falls back to StringIO and regular string for Python 2
outbuffer = BytesIO(b('<?xml version="1.0" encoding="UTF-8"?>\n'))
ETree.ElementTree(xml_root).write(outbuffer)
outbuffer.seek(0)
log.debug("Data for request: %s", outbuffer.read())
outbuffer.seek(0)
response = self.raw_request(
method,
uri,
data=outbuffer.getvalue(),
headers={'Content-Type': 'application/xml'})
outbuffer.close()
xml_response_root = ETree.XML(
response.content) if response.content else None
if self.log_requests:
request_elapsed_seconds = time.perf_counter(
) - request_start_seconds
log.info("clarity request method: '%s' uri: %s took: %.3f s",
method, uri, request_elapsed_seconds)
return xml_response_root
def check_pickle(obj):
fh = BytesIO()
cPickle.dump(obj, fh, protocol=cPickle.HIGHEST_PROTOCOL)
plen = fh.tell()
fh.seek(0, 0)
res = cPickle.load(fh)
fh.close()
return res, plen
def image_to_png(img):
"""Convert a PIL image to a PNG binary string."""
exp = BytesIO()
img.save(exp, format='png')
exp.seek(0)
s = exp.read()
exp.close()
return s
def test_safe_md5(self):
exp = 'ab07acbb1e496801937adfa772424bf7'
fd = BytesIO(b'foo bar baz')
obs = safe_md5(fd)
self.assertEqual(obs.hexdigest(), exp)
fd.close()
def image_to_png(img):
"""Convert a PIL image to a PNG binary string."""
exp = BytesIO()
img.save(exp, format='png')
exp.seek(0)
s = exp.read()
exp.close()
return s
def test_safe_md5(self):
exp = 'ab07acbb1e496801937adfa772424bf7'
fd = BytesIO(b'foo bar baz')
obs = safe_md5(fd)
self.assertEqual(obs.hexdigest(), exp)
fd.close()
def test_safe_md5(self):
"""Make sure we have the expected md5"""
exp = 'ab07acbb1e496801937adfa772424bf7'
fd = BytesIO(b'foo bar baz')
obs = safe_md5(fd)
self.assertEqual(obs.hexdigest(), exp)
fd.close()
def test_safe_md5(self):
"""Make sure we have the expected md5"""
exp = 'ab07acbb1e496801937adfa772424bf7'
fd = BytesIO(b'foo bar baz')
obs = safe_md5(fd)
self.assertEqual(obs.hexdigest(), exp)
fd.close()
def getBytes(tx):
"""
Hash transaction object into bytes data.
Argument:
tx (dict) -- transaction object
Return bytes sequence
"""
buf = BytesIO()
# write type and timestamp
pack("<bi", buf, (tx["type"], int(tx["timestamp"])))
# write senderPublicKey as bytes in buffer
if "senderPublicKey" in tx:
pack_bytes(buf, unhexlify(tx["senderPublicKey"]))
# if there is a requesterPublicKey
if "requesterPublicKey" in tx:
pack_bytes(buf, unhexlify(tx["requesterPublicKey"]))
# if there is a recipientId
if tx.get("recipientId", False):
recipientId = tx["recipientId"]
recipientId = base58.b58decode_check(str(recipientId) if not isinstance(recipientId, bytes) \
else recipientId)
else:
recipientId = b"\x00" * 21
pack_bytes(buf, recipientId)
# if there is a vendorField
if tx.get("vendorField", False):
vendorField = tx["vendorField"][:64].ljust(64, "\x00")
else:
vendorField = "\x00" * 64
pack_bytes(buf, vendorField.encode("utf-8"))
# write amount and fee value
pack("<QQ", buf, (int(tx["amount"]), int(tx["fee"])))
# if there is asset data
if tx.get("asset", False):
asset = tx["asset"]
typ = tx["type"]
if typ == 1 and "signature" in asset:
pack_bytes(buf, unhexlify(asset["signature"]["publicKey"]))
elif typ == 2 and "delegate" in asset:
pack_bytes(buf, asset["delegate"]["username"].encode("utf-8"))
elif typ == 3 and "votes" in asset:
pack_bytes(buf, "".join(asset["votes"]).encode("utf-8"))
else:
pass
# if there is a signature
if tx.get("signature", False):
pack_bytes(buf, unhexlify(tx["signature"]))
# if there is a second signature
if tx.get("signSignature", False):
pack_bytes(buf, unhexlify(tx["signSignature"]))
result = buf.getvalue()
buf.close()
return result
def check_binary(name, file_path=True):
# Handles files if file_path is True or text if file_path is False
if file_path:
temp = open(name, "rb")
else:
temp = BytesIO(name)
try:
return util.is_binary(temp.read(1024))
finally:
temp.close()
def test_50_get(self):
io = BytesIO()
self.webdav.download('handler.py', io)
self.assertEqual(utils.text(inspect.getsource(data_handler)), utils.text(io.getvalue()))
io.close()
io = BytesIO()
self.webdav.download('sample_handler.py', io)
self.assertEqual(utils.text(inspect.getsource(data_sample_handler)), utils.text(io.getvalue()))
io.close()
def make_image(image, height, width, channel):
"""Convert an numpy representation image to Image protobuf"""
output = BytesIO()
image.save(output, format='PNG')
image_string = output.getvalue()
output.close()
return Summary.Image(height=height,
width=width,
colorspace=channel,
encoded_image_string=image_string)
def test_50_get(self):
io = BytesIO()
self.webdav.download('handler.py', io)
self.assertEqual(inspect.getsource(data_handler), io.getvalue())
io.close()
io = BytesIO()
self.webdav.download('sample_handler.py', io)
self.assertEqual(inspect.getsource(data_sample_handler), io.getvalue())
io.close()
def check_binary(name, file_path=True):
# Handles files if file_path is True or text if file_path is False
if file_path:
temp = open(name, "rb")
else:
temp = BytesIO(name)
try:
return util.is_binary(temp.read(1024))
finally:
temp.close()
def xml_c14nize(data):
""" Returns a canonical value of an XML document.
"""
if not isinstance(data, etree._Element):
data = etree.fromstring(data.encode('utf-8'))
out = BytesIO()
data.getroottree().write_c14n(out)
value = out.getvalue()
out.close()
return value
def xml_c14nize(data):
""" Returns a canonical value of an XML document.
"""
if not isinstance(data, etree._Element):
data = etree.fromstring(data.encode('utf-8'))
out = BytesIO()
data.getroottree().write_c14n(out)
value = out.getvalue()
out.close()
return value
def test_50_get(self):
import easywebdav
with self.assertRaises(easywebdav.OperationFailed):
io = BytesIO()
self.webdav.download('handler.py', io)
io.close()
io = BytesIO()
self.webdav_up.download('handler.py', io)
self.assertEqual(utils.text(inspect.getsource(data_handler)), utils.text(io.getvalue()))
io.close()
def test_50_get(self):
import easywebdav
with self.assertRaises(easywebdav.OperationFailed):
io = BytesIO()
self.webdav.download('handler.py', io)
io.close()
io = BytesIO()
self.webdav_up.download('handler.py', io)
self.assertEqual(utils.text(inspect.getsource(data_handler)), utils.text(io.getvalue()))
io.close()
def generate(self):
"""Generates the Gzipped sitemap."""
xml = self.template()
fp = BytesIO()
gzip = GzipFile(self.filename, 'wb', 9, fp)
if isinstance(xml, six.text_type):
xml = xml.encode('utf8')
gzip.write(xml)
gzip.close()
data = fp.getvalue()
fp.close()
return data
def generate(self):
"""Generates the Gzipped sitemap."""
xml = self.template()
fp = BytesIO()
gzip = GzipFile(self.filename, 'wb', 9, fp)
if isinstance(xml, six.text_type):
xml = xml.encode('utf8')
gzip.write(xml)
gzip.close()
data = fp.getvalue()
fp.close()
return data
def get_qr_svg_code(totp_uri):
'''Get SVG code to display Qrcode for OTP.'''
url = qrcreate(totp_uri)
svg = ''
stream = BytesIO()
try:
url.svg(stream, scale=4, background="#eee", module_color="#222")
svg = stream.getvalue().decode().replace('\n', '')
svg = b64encode(svg.encode())
finally:
stream.close()
return svg
class QiniuFile(File):
def __init__(self, name, storage, mode):
self._storage = storage
if name.startswith(self._storage.location):
name = name[len(self._storage.location):]
self._name = name.lstrip('/')
self._mode = mode
self.file = BytesIO()
self._is_dirty = False
self._is_read = False
@property
def size(self):
if self._is_dirty or self._is_read:
# Get the size of a file like object
# Check http://stackoverflow.com/a/19079887
old_file_position = self.file.tell()
self.file.seek(0, os.SEEK_END)
self._size = self.file.tell()
self.file.seek(old_file_position, os.SEEK_SET)
if not hasattr(self, '_size'):
self._size = self._storage.size(self._name)
return self._size
def read(self, num_bytes=None):
if not self._is_read:
content = self._storage._read(self._name)
self.file = BytesIO(content)
self._is_read = True
if num_bytes is None:
data = self.file.read()
else:
data = self.file.read(num_bytes)
if 'b' in self._mode:
return data
else:
return force_text(data)
def write(self, content):
if 'w' not in self._mode:
raise AttributeError("File was opened for read-only access.")
self.file.write(force_bytes(content))
self._is_dirty = True
self._is_read = True
def close(self):
if self._is_dirty:
self.file.seek(0)
self._storage._save(self._name, self.file)
self.file.close()
def get_qr_svg_code(totp_uri):
'''Get SVG code to display Qrcode for OTP.'''
url = qrcreate(totp_uri)
svg = ''
stream = BytesIO()
try:
url.svg(stream, scale=4, background="#eee", module_color="#222")
svg = stream.getvalue().decode().replace('\n', '')
svg = b64encode(svg.encode())
finally:
stream.close()
return svg
class UniversalBytesIO(object):
def __init__(self, container=None, charset=None):
self.charset = charset or settings.DEFAULT_CHARSET
self._container = BytesIO() if container is None else container
# These methods partially implement the file-like object interface.
# See https://docs.python.org/3/library/io.html#io.IOBase
def close(self):
self._container.close()
def write(self, content):
self._container.write(self.make_bytes(content))
def flush(self):
self._container.flush()
def tell(self):
return self._container.tell()
def readable(self):
return False
def seekable(self):
return False
def writable(self):
return True
def writelines(self, lines):
for line in lines:
self.write(line)
def make_bytes(self, value):
"""Turn a value into a bytestring encoded in the output charset."""
if isinstance(value, bytes):
return bytes(value)
if isinstance(value, six.text_type):
return bytes(value.encode(self.charset))
# Handle non-string types
return force_bytes(value, self.charset)
def get_string_value(self):
return self._container.getvalue().decode(self.charset)
def getvalue(self):
return self._container.getvalue()
if sys.version_info[0:2] < (3, 5):
def seek(self, *args, **kwargs):
pass
def generateErrorMessagePlot(msgText):
"""
It creates a plot whith a specific error message
:param str msgText: the text which will appear on the plot.
:return: the plot.
"""
fn = BytesIO()
textGraph(msgText, fn, {})
data = fn.getvalue()
fn.close()
return S_OK(data)
def _read_file(self, file_path, is_inner=False):
try:
output = BytesIO(self.read(file_path))
result = output.getvalue()
output.close()
except KeyError:
try:
if is_inner:
raise KeyError
result = self._get_inner_zip()._read_file(file_path, True)
except KeyError:
raise Exception(_('Unable to find file "%s" in manifest.') % file_path)
return result
class TMemoryBuffer(TTransportBase, CReadableTransport):
"""Wraps a cStringIO object as a TTransport.
NOTE: Unlike the C++ version of this class, you cannot write to it
then immediately read from it. If you want to read from a
TMemoryBuffer, you must either pass a string to the constructor.
TODO(dreiss): Make this work like the C++ version.
"""
def __init__(self, value=None):
"""value -- a value to read from for stringio
If value is set, this will be a transport for reading,
otherwise, it is for writing"""
if value is not None:
self._buffer = BytesIO(value)
else:
self._buffer = BytesIO()
def isOpen(self):
return not self._buffer.closed
def open(self):
pass
def close(self):
self._buffer.close()
def read(self, sz):
return self._buffer.read(sz)
def write(self, buf):
try:
self._buffer.write(buf)
except TypeError:
self._buffer.write(buf.encode('cp437'))
def flush(self):
pass
def getvalue(self):
return self._buffer.getvalue()
# Implement the CReadableTransport interface.
@property
def cstringio_buf(self):
return self._buffer
def cstringio_refill(self, partialread, reqlen):
# only one shot at reading...
raise EOFError()
def tobuffer(kv):
firstkey = None
buf = BytesIO()
for k, v in kv:
if firstkey is None:
firstkey = k
buf.write(v.tostring())
val = buf.getvalue()
buf.close()
if firstkey is None:
return iter([])
else:
label = prefix + '-' + getlabel(firstkey) + ".bin"
return iter([(label, val)])
def uploadChunk(self, upload, chunk):
"""
Appends the chunk into the temporary file.
"""
# If we know the chunk size is too large or small, fail early.
self.checkUploadSize(upload, self.getChunkSize(chunk))
if isinstance(chunk, six.text_type):
chunk = chunk.encode('utf8')
if isinstance(chunk, six.binary_type):
chunk = BytesIO(chunk)
# Restore the internal state of the streaming SHA-512 checksum
checksum = _hash_state.restoreHex(upload['sha512state'], 'sha512')
if self.requestOffset(upload) > upload['received']:
# This probably means the server died midway through writing last
# chunk to disk, and the database record was not updated. This
# means we need to update the sha512 state with the difference.
with open(upload['tempFile'], 'rb') as tempFile:
tempFile.seek(upload['received'])
while True:
data = tempFile.read(BUF_SIZE)
if not data:
break
checksum.update(data)
with open(upload['tempFile'], 'a+b') as tempFile:
size = 0
while not upload['received'] + size > upload['size']:
data = chunk.read(BUF_SIZE)
if not data:
break
size += len(data)
tempFile.write(data)
checksum.update(data)
chunk.close()
try:
self.checkUploadSize(upload, size)
except ValidationException:
with open(upload['tempFile'], 'a+b') as tempFile:
tempFile.truncate(upload['received'])
raise
# Persist the internal state of the checksum
upload['sha512state'] = _hash_state.serializeHex(checksum)
upload['received'] += size
return upload
def decompress(data):
"""
Decompresses given data via the ``gzip`` module.
Decoding is left as an exercise for the client code.
"""
buff = BytesIO(data)
with gzip.GzipFile(fileobj=buff, mode='r') as fd:
result = fd.read()
buff.close()
return result
def response_to_json(response):
"""
hack for requests in python 2.6
"""
content = response.content
# hack for requests in python 2.6
if 'application/json' in response.headers['Content-Type']:
if content[:2] == '\x1f\x8b': # gzip file magic header
f = BytesIO(content)
g = gzip.GzipFile(fileobj=f)
content = g.read()
g.close()
f.close()
return json.loads(str(content.decode('utf-8')))
def test_writing_0_records(self):
expected = r"""
<?xml version="1.0" encoding="UTF-8"?>
<collection xmlns="http://www.loc.gov/MARC21/slim">
</collection>
"""
expected = textwrap.dedent(expected[1:]).replace('\n', '')
if str != binary_type:
expected = expected.encode()
file_handle = BytesIO()
try:
writer = pymarc.XMLWriter(file_handle)
writer.close(close_fh=False)
self.assertEquals(file_handle.getvalue(), expected)
finally:
file_handle.close()
def visit_immutation(self, node, children):
context = self._final_context()
child_type = children[0].expr_name
if child_type == 'preview':
if self.tool == 'httpie':
command = ['http'] + context.httpie_args(self.method,
quote=True)
else:
assert self.tool == 'curl'
command = ['curl'] + context.curl_args(self.method, quote=True)
click.echo(' '.join(command))
elif child_type == 'action':
output = BytesIO()
try:
env = Environment(stdout=output, is_windows=False)
# XXX: httpie_main() doesn't provide an API for us to get the
# HTTP response object, so we use this super dirty hack -
# sys.settrace() to intercept get_response() that is called in
# httpie_main() internally. The HTTP response intercepted is
# assigned to self.last_response, which may be useful for
# self.listener.
sys.settrace(self._trace_get_response)
try:
httpie_main(context.httpie_args(self.method), env=env)
finally:
sys.settrace(None)
content = output.getvalue()
finally:
output.close()
# XXX: Work around a bug of click.echo_via_pager(). When you pass
# a bytestring to echo_via_pager(), it converts the bytestring with
# str(b'abc'), which makes it "b'abc'".
if six.PY2:
content = unicode(content, 'utf-8') # noqa
else:
content = str(content, 'utf-8')
click.echo_via_pager(content)
if self.last_response:
self.listener.response_returned(self.context,
self.last_response)
return node
def compress(data):
"""
Compresses a given bit of data via the ``gzip`` stdlib module.
.. note::
This assumes the given data is a byte string, already decoded.
"""
buff = BytesIO()
with gzip.GzipFile(fileobj=buff, mode='w') as fd:
fd.write(data)
buff.seek(0)
result = buff.read()
buff.close()
return result
