def transfer(self, filename, fileobj=None, xml=None):
fields = {}
if fileobj is None and filename is None:
raise BQCommError('Filename or fileobj are required for transfer')
if fileobj is None and os.path.exists(filename):
fileobj = open(filename, 'rb')
if fileobj is not None and filename is None:
filename = fileobj.name
if fileobj is not None:
filename = normalize_unicode(filename)
fields['file'] = (os.path.basename(filename), fileobj,
'application/octet-stream')
if xml is not None:
fields['file_resource'] = xml
if fields:
# https://github.com/requests/toolbelt/issues/75
m = MultipartEncoder(fields=fields)
m._read = m.read #pylint: disable=protected-access
m.read = lambda size: m._read(8129 * 1024) # 8MB
# ID generator is used to force load balancing operations
response = self.post("transfer_" + id_generator(),
data=m,
headers={
'Accept': 'text/xml',
'Content-Type': m.content_type
})
return response
def test_encoding(self):
"""Test MultipartEncoder encoding"""
with SubsetIO(self.fd, 1, 64) as fd:
encoder = MultipartEncoder(
fields={'file': ("filename", fd, 'application/octet-stream')})
data = encoder.read()
self.assertTrue(len(data) > 0)
def test_encoding(self):
"""Test MultipartEncoder encoding"""
with SubsetIO(self.fd, 1, 64) as fd:
encoder = MultipartEncoder(
fields={'file': (
"filename", fd, 'application/octet-stream'
)}
)
data = encoder.read()
self.assertTrue(len(data) > 0)
def test_multipart_encoder_passthru():
wrapper = FilePassthru("tests/data/files/test_R1_L001.fq.bz2")
wrapper_len = len(wrapper.read())
wrapper.seek(0)
assert wrapper_len == wrapper._fsize
multipart_fields = OrderedDict()
multipart_fields["file"] = ("fakefile", wrapper, "application/x-gzip")
encoder = MultipartEncoder(multipart_fields)
MAGIC_HEADER_LEN = 178
wrapper.seek(0)
assert len(encoder.read()) - MAGIC_HEADER_LEN == wrapper_len
def test_multipart_encoder():
long_seq = b'ACGT' * 50
data = b'\n'.join(b'>header_' + str(i).encode() + b'\n' + long_seq + b'\n'
for i in range(200))
wrapper = FASTXTranslator(BytesIO(data), recompress=False)
wrapper_len = len(wrapper.read())
wrapper.seek(0)
multipart_fields = OrderedDict()
multipart_fields['file'] = ('fakefile', wrapper, 'application/x-gzip')
encoder = MultipartEncoder(multipart_fields)
MAGIC_HEADER_LEN = 178
wrapper.seek(0)
assert len(encoder.read()) - MAGIC_HEADER_LEN == wrapper_len
def test_multipart_encoder_interleave():
pair = ("tests/data/files/test_R1_L001.fq.gz", "tests/data/files/test_R2_L001.fq.gz")
fname, fsize, fformat = _file_stats(pair)
wrapper = FASTXInterleave(pair, fsize, fformat)
wrappertext = wrapper.read()
wrapper.seek(0)
assert len(wrappertext) == fsize
multipart_fields = OrderedDict()
multipart_fields["file"] = ("fakefile", wrapper, "text/plain")
encoder = MultipartEncoder(multipart_fields)
MAGIC_HEADER_LEN = 170 # shorter because of text/plain mime-type
encodertext = encoder.read()
assert len(encodertext) - MAGIC_HEADER_LEN == len(wrappertext)
def test_multipart_encoder_interleave():
pair = ("tests/data/files/test_R1_L001.fq.gz",
"tests/data/files/test_R2_L001.fq.gz")
fname, fsize, fformat = _file_stats(pair)
wrapper = FASTXInterleave(pair, fsize, fformat)
wrappertext = wrapper.read()
wrapper.seek(0)
assert len(wrappertext) == fsize
multipart_fields = OrderedDict()
multipart_fields["file"] = ("fakefile", wrapper, "text/plain")
encoder = MultipartEncoder(multipart_fields)
MAGIC_HEADER_LEN = 170 # shorter because of text/plain mime-type
encodertext = encoder.read()
assert len(encodertext) - MAGIC_HEADER_LEN == len(wrappertext)
def test_multipart_encoder_passthru():
wrapper = FilePassthru(
"tests/data/files/test_R1_L001.fq.bz2",
os.path.getsize("tests/data/files/test_R1_L001.fq.bz2"),
)
wrapper_len = len(wrapper.read())
wrapper.seek(0)
assert wrapper_len == wrapper._fsize
multipart_fields = OrderedDict()
multipart_fields["file"] = ("fakefile", wrapper, "application/x-gzip")
encoder = MultipartEncoder(multipart_fields)
MAGIC_HEADER_LEN = 178
wrapper.seek(0)
assert len(encoder.read()) - MAGIC_HEADER_LEN == wrapper_len
