def _upload_form_encoded(self,
url,
filepath,
shard_size,
start_pos,
read_size=1024,
callback=None):
""" Streams file from disk and uploads it.
:param url: API endpoint as URL to upload to
:param filepath: Path to file as string
:return: requests.Response
"""
validpath = self.check_path(filepath)
if shard_size == 0:
shard_size = SizeHelpers.mib_to_bytes(250)
shard = ShardFile(validpath,
'rb',
shard_size=shard_size,
start_pos=start_pos,
read_size=read_size,
callback=callback)
m = MultipartEncoder({'file': ('file', shard)})
headers = {'Content-Type': m.content_type}
return requests.post(url, data=m, headers=headers)
def test_init_callback(self):
test_shard = ShardFile(self.testfile, 'rb', callback=callback)
self.assertEqual(test_shard.callback, callback)
def test_context_manager(self):
with ShardFile(self.testfile) as shard:
self.assertTrue(isinstance(shard, ShardFile))
self.new_shard = shard
self.assertTrue(self.new_shard._f_obj.closed)
del self.new_shard
def setUp(self):
self.testfile = 'tests/one-meg.testfile'
self.shard = ShardFile(self.testfile, 'rb')
class TestShardFile(unittest.TestCase):
def setUp(self):
self.testfile = 'tests/one-meg.testfile'
self.shard = ShardFile(self.testfile, 'rb')
def tearDown(self):
del self.shard
def test_instantiation(self):
self.assertTrue(self.shard._f_obj)
self.assertTrue(SizeHelpers.mib_to_bytes(25), self.shard.shard_size)
self.assertEqual(self.shard._f_obj.tell(), 0)
self.assertEqual(self.shard.filesize, SizeHelpers.mib_to_bytes(1))
self.assertEqual(self.shard.max_seek, SizeHelpers.mib_to_bytes(1))
self.assertFalse(hasattr(self.shard, 'callback'))
def test_iter(self):
result = self.shard.__iter__()
self.assertTrue(isinstance(result, types.GeneratorType))
def test_context_manager(self):
with ShardFile(self.testfile) as shard:
self.assertTrue(isinstance(shard, ShardFile))
self.new_shard = shard
self.assertTrue(self.new_shard._f_obj.closed)
del self.new_shard
def test_next(self):
result = self.shard.__next__()
self.assertTrue(result)
with self.assertRaises(StopIteration):
while True:
self.shard.__next__()
def test_init_callback(self):
test_shard = ShardFile(self.testfile, 'rb', callback=callback)
self.assertEqual(test_shard.callback, callback)
def test__len__(self):
fivetwelvekay = SizeHelpers.kib_to_bytes(512)
self.shard.read(fivetwelvekay)
self.assertTrue(self.shard.__len__() < self.shard.max_seek)
def test_read_with_size(self):
fivetwelvekay = SizeHelpers.kib_to_bytes(512)
read = self.shard.read(fivetwelvekay)
self.assertEqual(fivetwelvekay, len(read))
def test_read_no_size(self):
read = self.shard.read()
self.assertEqual(len(read), SizeHelpers.mib_to_bytes(1))
def test_read_loc_is_maxseek(self):
self.shard.read(self.shard.max_seek)
read_more = self.shard.read(1)
self.assertEqual(read_more, '')
def test_seek(self):
number = random.randint(1, SizeHelpers.kib_to_bytes(768))
self.shard.seek(number)
self.assertEqual(self.shard._f_obj.tell(), number)
def test_tell(self):
number = random.randint(1, SizeHelpers.kib_to_bytes(768))
self.shard.seek(number)
self.assertEqual(self.shard.tell(), number)
def test_close(self):
self.shard._slicegen = None
self.shard.close()
self.assertFalse(hasattr(self.shard, '_slicegen'))
self.assertTrue(self.shard._f_obj.closed)
def test_generate_slices(self):
gen = self.shard._generate_slices()
self.assertTrue(isinstance(gen, types.GeneratorType))
def test_generate_slices_at_max_seek(self):
gen = self.shard._generate_slices()
self.shard.seek(SizeHelpers.mib_to_bytes(1))
for slice in gen:
self.assertIs(slice, None)
def test_generate_slices_near_end(self):
gen = self.shard._generate_slices()
self.shard.seek(SizeHelpers.mib_to_bytes(1) - 10)
for slice in gen:
self.assertEqual(len(slice), 10)
def test_generate_slice_read_size(self):
gen = self.shard._generate_slices()
for slice in gen:
self.assertEqual(len(slice), self.shard.read_size)
break
def test_generate_slices_callback(self):
gen = self.shard._generate_slices()
next(gen)
self.shard.callback = callback
next(gen)
def test_calc_max_seek_near_end(self):
self.shard.seek(SizeHelpers.mib_to_bytes(1) - 10)
self.shard._calc_max_seek()
self.assertEqual(self.shard.max_seek, self.shard.filesize)
def test_calc_max_seek_beginning_small_shard(self):
self.shard.shard_size = SizeHelpers.kib_to_bytes(512)
self.shard._calc_max_seek()
self.assertEqual(self.shard.max_seek, self.shard.shard_size)
def test_calc_total_read(self):
number = random.randint(1, SizeHelpers.mib_to_bytes(1))
self.shard.seek(number)
self.shard._calc_total_read()
self.assertEqual(self.shard.total_read_bytes,
self.shard.max_seek - number)
def test_size_helpers_bytes_to_kib(self):
self.assertEqual(SizeHelpers.bytes_to_kib(1024), 1)
def test_size_helpers_bytes_to_mib(self):
self.assertEqual(SizeHelpers.bytes_to_mib(1048576), 1)
def setUp(self):
self.testfile = 'tests/one-meg.testfile'
self.shard = ShardFile(self.testfile, 'rb')
class TestShardFile(unittest.TestCase):
def setUp(self):
self.testfile = 'tests/one-meg.testfile'
self.shard = ShardFile(self.testfile, 'rb')
def tearDown(self):
del self.shard
def test_instantiation(self):
self.assertTrue(self.shard._f_obj)
self.assertTrue(SizeHelpers.mib_to_bytes(25), self.shard.shard_size)
self.assertEqual(self.shard._f_obj.tell(), 0)
self.assertEqual(self.shard.filesize, SizeHelpers.mib_to_bytes(1))
self.assertEqual(self.shard.max_seek, SizeHelpers.mib_to_bytes(1))
self.assertFalse(hasattr(self.shard, 'callback'))
def test_iter(self):
result = self.shard.__iter__()
self.assertTrue(isinstance(result, types.GeneratorType))
def test_context_manager(self):
with ShardFile(self.testfile) as shard:
self.assertTrue(isinstance(shard, ShardFile))
self.new_shard = shard
self.assertTrue(self.new_shard._f_obj.closed)
del self.new_shard
def test_next(self):
result = self.shard.__next__()
self.assertTrue(result)
with self.assertRaises(StopIteration):
while True:
self.shard.__next__()
def test_init_callback(self):
test_shard = ShardFile(self.testfile, 'rb', callback=callback)
self.assertEqual(test_shard.callback, callback)
def test__len__(self):
fivetwelvekay = SizeHelpers.kib_to_bytes(512)
self.shard.read(fivetwelvekay)
self.assertTrue(self.shard.__len__() < self.shard.max_seek)
def test_read_with_size(self):
fivetwelvekay = SizeHelpers.kib_to_bytes(512)
read = self.shard.read(fivetwelvekay)
self.assertEqual(fivetwelvekay, len(read))
def test_read_no_size(self):
read = self.shard.read()
self.assertEqual(len(read), SizeHelpers.mib_to_bytes(1))
def test_read_loc_is_maxseek(self):
self.shard.read(self.shard.max_seek)
read_more = self.shard.read(1)
self.assertEqual(read_more, '')
def test_seek(self):
number = random.randint(1, SizeHelpers.kib_to_bytes(768))
self.shard.seek(number)
self.assertEqual(self.shard._f_obj.tell(), number)
def test_tell(self):
number = random.randint(1, SizeHelpers.kib_to_bytes(768))
self.shard.seek(number)
self.assertEqual(self.shard.tell(), number)
def test_close(self):
self.shard._slicegen = None
self.shard.close()
self.assertFalse(hasattr(self.shard, '_slicegen'))
self.assertTrue(self.shard._f_obj.closed)
def test_generate_slices(self):
gen = self.shard._generate_slices()
self.assertTrue(isinstance(gen, types.GeneratorType))
def test_generate_slices_at_max_seek(self):
gen = self.shard._generate_slices()
self.shard.seek(SizeHelpers.mib_to_bytes(1))
for slice in gen:
self.assertIs(slice, None)
def test_generate_slices_near_end(self):
gen = self.shard._generate_slices()
self.shard.seek(SizeHelpers.mib_to_bytes(1) - 10)
for slice in gen:
self.assertEqual(len(slice), 10)
def test_generate_slice_read_size(self):
gen = self.shard._generate_slices()
for slice in gen:
self.assertEqual(len(slice), self.shard.read_size)
break
def test_generate_slices_callback(self):
gen = self.shard._generate_slices()
next(gen)
self.shard.callback = callback
next(gen)
def test_calc_max_seek_near_end(self):
self.shard.seek(SizeHelpers.mib_to_bytes(1) - 10)
self.shard._calc_max_seek()
self.assertEqual(self.shard.max_seek, self.shard.filesize)
def test_calc_max_seek_beginning_small_shard(self):
self.shard.shard_size = SizeHelpers.kib_to_bytes(512)
self.shard._calc_max_seek()
self.assertEqual(self.shard.max_seek, self.shard.shard_size)
def test_calc_total_read(self):
number = random.randint(1, SizeHelpers.mib_to_bytes(1))
self.shard.seek(number)
self.shard._calc_total_read()
self.assertEqual(
self.shard.total_read_bytes, self.shard.max_seek - number)
def test_size_helpers_bytes_to_kib(self):
self.assertEqual(SizeHelpers.bytes_to_kib(1024), 1)
def test_size_helpers_bytes_to_mib(self):
self.assertEqual(SizeHelpers.bytes_to_mib(1048576), 1)
